{"markup":"\u003C?xml version=\u00221.0\u0022 encoding=\u0022UTF-8\u0022 ?\u003E\n    \u003Chtml version=\u0022HTML+RDFa+MathML 1.1\u0022\n    xmlns:content=\u0022http:\/\/purl.org\/rss\/1.0\/modules\/content\/\u0022\n    xmlns:dc=\u0022http:\/\/purl.org\/dc\/terms\/\u0022\n    xmlns:foaf=\u0022http:\/\/xmlns.com\/foaf\/0.1\/\u0022\n    xmlns:og=\u0022http:\/\/ogp.me\/ns#\u0022\n    xmlns:rdfs=\u0022http:\/\/www.w3.org\/2000\/01\/rdf-schema#\u0022\n    xmlns:sioc=\u0022http:\/\/rdfs.org\/sioc\/ns#\u0022\n    xmlns:sioct=\u0022http:\/\/rdfs.org\/sioc\/types#\u0022\n    xmlns:skos=\u0022http:\/\/www.w3.org\/2004\/02\/skos\/core#\u0022\n    xmlns:xsd=\u0022http:\/\/www.w3.org\/2001\/XMLSchema#\u0022\n    xmlns:mml=\u0022http:\/\/www.w3.org\/1998\/Math\/MathML\u0022\u003E\n  \u003Chead\u003E\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022\/\/cdn.jsdelivr.net\/qtip2\/2.2.1\/jquery.qtip.min.js\u0022\u003E\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/js\/js_YjAJQgxDlFX6S-O02jj9jCrVbrwlY3CGgCg1FzPlvBs.js\u0022\u003E\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022\u003E\n\u003C!--\/\/--\u003E\u003C![CDATA[\/\/\u003E\u003C!--\nif(typeof window.MathJax === \u0022undefined\u0022) window.MathJax = { menuSettings: { zoom: \u0022Click\u0022 } };\n\/\/--\u003E\u003C!]]\u003E\n\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/js\/js_waP91NpgGpectm_6Y2XDEauLJ8WCSCBKmmA87unpp2E.js\u0022\u003E\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022https:\/\/www.googletagmanager.com\/gtag\/js?id=G-0K57TCX5BY\u0022\u003E\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022\u003E\n\u003C!--\/\/--\u003E\u003C![CDATA[\/\/\u003E\u003C!--\nwindow.dataLayer = window.dataLayer || [];function gtag(){dataLayer.push(arguments)};gtag(\u0022js\u0022, new Date());gtag(\u0022set\u0022, \u0022developer_id.dMDhkMT\u0022, true);gtag(\u0022config\u0022, \u0022G-0K57TCX5BY\u0022, {\u0022groups\u0022:\u0022default\u0022,\u0022anonymize_ip\u0022:true});\n\/\/--\u003E\u003C!]]\u003E\n\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022\u003E\n\u003C!--\/\/--\u003E\u003C![CDATA[\/\/\u003E\u003C!--\njQuery.extend(Drupal.settings, {\u0022basePath\u0022:\u0022\\\/\u0022,\u0022pathPrefix\u0022:\u0022\u0022,\u0022highwire\u0022:{\u0022ac\u0022:{\u0022medrxiv;2024.06.27.24309562v1\u0022:{\u0022access\u0022:{\u0022full\u0022:true},\u0022pisa_id\u0022:\u0022medrxiv;2024.06.27.24309562v1\u0022,\u0022apath\u0022:\u0022\u0022,\u0022jcode\u0022:\u0022medrxiv\u0022}},\u0022processed\u0022:[\u0022highwire_math\u0022],\u0022markup\u0022:[{\u0022requested\u0022:\u0022full-text\u0022,\u0022variant\u0022:\u0022full-text\u0022,\u0022view\u0022:\u0022full\u0022,\u0022pisa\u0022:\u0022medrxiv;2024.06.27.24309562v1\u0022}]},\u0022instances\u0022:\u0022{\\u0022highwire_abstract_tooltip\\u0022:{\\u0022content\\u0022:{\\u0022text\\u0022:\\u0022\\u0022},\\u0022style\\u0022:{\\u0022tip\\u0022:{\\u0022width\\u0022:20,\\u0022height\\u0022:20,\\u0022border\\u0022:1,\\u0022offset\\u0022:0,\\u0022corner\\u0022:true},\\u0022classes\\u0022:\\u0022qtip-custom hw-tooltip hw-abstract-tooltip qtip-shadow qtip-rounded\\u0022,\\u0022classes_custom\\u0022:\\u0022hw-tooltip hw-abstract-tooltip\\u0022},\\u0022position\\u0022:{\\u0022at\\u0022:\\u0022right center\\u0022,\\u0022my\\u0022:\\u0022left center\\u0022,\\u0022viewport\\u0022:true,\\u0022adjust\\u0022:{\\u0022method\\u0022:\\u0022shift\\u0022}},\\u0022show\\u0022:{\\u0022event\\u0022:\\u0022mouseenter click \\u0022,\\u0022solo\\u0022:true},\\u0022hide\\u0022:{\\u0022event\\u0022:\\u0022mouseleave \\u0022,\\u0022fixed\\u0022:1,\\u0022delay\\u0022:\\u0022100\\u0022}},\\u0022highwire_author_tooltip\\u0022:{\\u0022content\\u0022:{\\u0022text\\u0022:\\u0022\\u0022},\\u0022style\\u0022:{\\u0022tip\\u0022:{\\u0022width\\u0022:15,\\u0022height\\u0022:15,\\u0022border\\u0022:1,\\u0022offset\\u0022:0,\\u0022corner\\u0022:true},\\u0022classes\\u0022:\\u0022qtip-custom hw-tooltip hw-author-tooltip qtip-shadow qtip-rounded\\u0022,\\u0022classes_custom\\u0022:\\u0022hw-tooltip hw-author-tooltip\\u0022},\\u0022position\\u0022:{\\u0022at\\u0022:\\u0022top center\\u0022,\\u0022my\\u0022:\\u0022bottom center\\u0022,\\u0022viewport\\u0022:true,\\u0022adjust\\u0022:{\\u0022method\\u0022:\\u0022\\u0022}},\\u0022show\\u0022:{\\u0022event\\u0022:\\u0022mouseenter \\u0022,\\u0022solo\\u0022:true},\\u0022hide\\u0022:{\\u0022event\\u0022:\\u0022mouseleave \\u0022,\\u0022fixed\\u0022:1,\\u0022delay\\u0022:\\u0022100\\u0022}},\\u0022highwire_reflinks_tooltip\\u0022:{\\u0022content\\u0022:{\\u0022text\\u0022:\\u0022\\u0022},\\u0022style\\u0022:{\\u0022tip\\u0022:{\\u0022width\\u0022:15,\\u0022height\\u0022:15,\\u0022border\\u0022:1,\\u0022mimic\\u0022:\\u0022top center\\u0022,\\u0022offset\\u0022:0,\\u0022corner\\u0022:true},\\u0022classes\\u0022:\\u0022qtip-custom hw-tooltip hw-ref-link-tooltip qtip-shadow qtip-rounded\\u0022,\\u0022classes_custom\\u0022:\\u0022hw-tooltip hw-ref-link-tooltip\\u0022},\\u0022position\\u0022:{\\u0022at\\u0022:\\u0022bottom left\\u0022,\\u0022my\\u0022:\\u0022top left\\u0022,\\u0022viewport\\u0022:true,\\u0022adjust\\u0022:{\\u0022method\\u0022:\\u0022flip\\u0022}},\\u0022show\\u0022:{\\u0022event\\u0022:\\u0022mouseenter \\u0022,\\u0022solo\\u0022:true},\\u0022hide\\u0022:{\\u0022event\\u0022:\\u0022mouseleave \\u0022,\\u0022fixed\\u0022:1,\\u0022delay\\u0022:\\u0022100\\u0022}}}\u0022,\u0022qtipDebug\u0022:\u0022{\\u0022leaveElement\\u0022:0}\u0022,\u0022googleanalytics\u0022:{\u0022account\u0022:[\u0022G-0K57TCX5BY\u0022],\u0022trackOutbound\u0022:1,\u0022trackMailto\u0022:1,\u0022trackDownload\u0022:1,\u0022trackDownloadExtensions\u0022:\u00227z|aac|arc|arj|asf|asx|avi|bin|csv|doc(x|m)?|dot(x|m)?|exe|flv|gif|gz|gzip|hqx|jar|jpe?g|js|mp(2|3|4|e?g)|mov(ie)?|msi|msp|pdf|phps|png|ppt(x|m)?|pot(x|m)?|pps(x|m)?|ppam|sld(x|m)?|thmx|qtm?|ra(m|r)?|sea|sit|tar|tgz|torrent|txt|wav|wma|wmv|wpd|xls(x|m|b)?|xlt(x|m)|xlam|xml|z|zip\u0022,\u0022trackColorbox\u0022:1},\u0022ajaxPageState\u0022:{\u0022js\u0022:{\u0022\\\/\\\/cdn.jsdelivr.net\\\/qtip2\\\/2.2.1\\\/jquery.qtip.min.js\u0022:1,\u0022sites\\\/all\\\/modules\\\/highwire\\\/highwire\\\/plugins\\\/highwire_markup_process\\\/js\\\/highwire_article_reference_popup.js\u0022:1,\u0022sites\\\/all\\\/modules\\\/highwire\\\/highwire\\\/plugins\\\/highwire_markup_process\\\/js\\\/highwire_at_symbol.js\u0022:1,\u00220\u0022:1,\u0022sites\\\/all\\\/modules\\\/contrib\\\/google_analytics\\\/googleanalytics.js\u0022:1,\u0022https:\\\/\\\/www.googletagmanager.com\\\/gtag\\\/js?id=G-0K57TCX5BY\u0022:1,\u00221\u0022:1}}});\n\/\/--\u003E\u003C!]]\u003E\n\u003C\/script\u003E\n\u003Clink type=\u0022text\/css\u0022 rel=\u0022stylesheet\u0022 href=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/advagg_css\/css__dn-cpI1YtkU_iLHgA5WhlkxgYWyat_IxjF_B-WSYrpE__a9hIbt0eaZ7d5nhwnm2weG8R_2eXK4EvoOx9dOxouHE__QrrGUc7CpljPR5Aph-ukPbcwtK4AWrHGwCEXJ_k1V_c.css\u0022 media=\u0022all\u0022 \/\u003E\n\u003Clink type=\u0022text\/css\u0022 rel=\u0022stylesheet\u0022 href=\u0022\/\/cdn.jsdelivr.net\/qtip2\/2.2.1\/jquery.qtip.min.css\u0022 media=\u0022all\u0022 \/\u003E\n\u003Clink type=\u0022text\/css\u0022 rel=\u0022stylesheet\u0022 href=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/advagg_css\/css__HGACIFBlu2o05y3afvqlt5wrE_5Dn6MXsexfuEpeIwg__t4SOPxucAPoV3Os7g8dXqyMB1HRXQridRJ82X7nE33E__QrrGUc7CpljPR5Aph-ukPbcwtK4AWrHGwCEXJ_k1V_c.css\u0022 media=\u0022all\u0022 \/\u003E\n\u003Clink rel=\u0027stylesheet\u0027 type=\u0027text\/css\u0027 href=\u0027\/sites\/all\/modules\/contrib\/panels\/plugins\/layouts\/onecol\/onecol.css\u0027 \/\u003E\u003C\/head\u003E\u003Cbody\u003E\u003Cdiv class=\u0022panels-ajax-tab-panel panels-ajax-tab-panel-article-tab-full-text\u0022\u003E\u003Cdiv class=\u0022panel-display panel-1col clearfix\u0022 \u003E\n  \u003Cdiv class=\u0022panel-panel panel-col\u0022\u003E\n    \u003Cdiv\u003E\u003Cdiv class=\u0022panel-pane pane-highwire-markup\u0022 \u003E\n  \n      \n  \n  \u003Cdiv class=\u0022pane-content\u0022\u003E\n    \u003Cdiv class=\u0022highwire-markup\u0022\u003E\u003Cdiv xmlns=\u0022http:\/\/www.w3.org\/1999\/xhtml\u0022 data-highwire-cite-ref-tooltip-instance=\u0022highwire_reflinks_tooltip\u0022 class=\u0022content-block-markup\u0022 xmlns:xhtml=\u0022http:\/\/www.w3.org\/1999\/xhtml\u0022\u003E\u003Cdiv class=\u0022article fulltext-view \u0022\u003E\u003Cspan class=\u0022highwire-journal-article-marker-start\u0022\u003E\u003C\/span\u003E\u003Cdiv class=\u0022section abstract\u0022 id=\u0022abstract-1\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EAbstract\u003C\/h2\u003E\u003Cp id=\u0022p-4\u0022\u003EWith an estimated 3 billion people globally lacking access to dermatological care, technological solutions leveraging artificial intelligence (AI) have been proposed to improve access\u003Csup\u003E\u003Ca id=\u0022xref-ref-1-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-1\u0022\u003E1\u003C\/a\u003E\u003C\/sup\u003E. Diagnostic AI algorithms, however, require high-quality datasets to allow development and testing, particularly those that enable evaluation of both unimodal and multimodal approaches. Currently, the majority of dermatology AI algorithms are built and tested on proprietary, siloed data, often from a single site and with only a single image type (i.e., clinical or dermoscopic). To address this, we developed and released the Melanoma Research Alliance Multimodal Image Dataset for AI-based Skin Cancer (MIDAS) dataset, the largest publicly available, prospectively-recruited, paired dermoscopic- and clinical image-based dataset of biopsy-proven and dermatopathology-labeled skin lesions. We explored model performance on real-world cases using four previously published state-of-the-art (SOTA) models and compared model-to-clinician diagnostic performance. We also assessed algorithm performance using clinical photography taken at different distances from the lesion to assess its influence across diagnostic categories.\u003C\/p\u003E\u003Cp id=\u0022p-5\u0022\u003EWe prospectively enrolled 796 patients through an IRB-approved protocol with informed consent representing 1290 unique lesions and 3830 total images (including dermoscopic and clinical images taken at 15-cm and 30-cm distance). Images represented the diagnostic diversity of lesions seen in general dermatology, with malignant, benign, and inflammatory lesions that included melanocytic nevi (22%; n=234), invasive cutaneous melanomas (4%; n=46), and melanoma in situ (4%; n=47). When evaluating SOTA models using the MIDAS dataset, we observed performance reduction across all models compared to their previously published performance metrics, indicating challenges to generalizability of current SOTA algorithms. As a comparative baseline, the dermatologists performing biopsies were 79% accurate with their top-1 diagnosis at differentiating a malignant from benign lesion. For malignant lesions, algorithms performed better on images acquired at 15-cm compared to 30-cm distance while dermoscopic images yielded higher sensitivity compared to clinical images.\u003C\/p\u003E\u003Cp id=\u0022p-6\u0022\u003EImproving our understanding of the strengths and weaknesses of AI diagnostic algorithms is critical as these tools advance towards widespread clinical deployment. While many algorithms may report high performance metrics, caution should be taken due to the potential for overfitting to localized datasets. MIDAS\u2019s robust, multimodal, and diverse dataset allows researchers to evaluate algorithms on our real-world images and better assess their generalizability.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-1\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E1. Introduction\u003C\/h2\u003E\u003Cp id=\u0022p-19\u0022\u003ESkin disease causes significant morbidity and mortality, and access to dermatological care is of global concern. Artificial intelligence (AI) has been proposed as a technological solution to improve skin disease diagnosis and triage. However, to date, most algorithms do not reflect the workflow of dermatologists.\u003C\/p\u003E\u003Cp id=\u0022p-20\u0022\u003EDermatologists assess skin lesions using both macroscopic clinical examinations and specialized dermoscopy imaging, while integrating key demographic and medical history data known to influence malignancy risk \u003Csup\u003E\u003Ca id=\u0022xref-ref-2-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-2\u0022\u003E2\u003C\/a\u003E,\u003Ca id=\u0022xref-ref-3-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-3\u0022\u003E3\u003C\/a\u003E\u003C\/sup\u003E. This multimodal approach may drive the improved performance of in-vivo assessment compared to dermoscopic photography. Despite this inherent multimodality, most dermatology artificial intelligence (AI) algorithms have been unimodal. For example, at the time of writing, prospective dermatology computer vision AI studies have focused exclusively on either clinical or dermoscopic images\u003Csup\u003E\u003Ca id=\u0022xref-ref-4-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-4\u0022\u003E4\u003C\/a\u003E\u2013\u003Ca id=\u0022xref-ref-7-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-7\u0022\u003E7\u003C\/a\u003E\u003C\/sup\u003E.\u003C\/p\u003E\u003Cp id=\u0022p-21\u0022\u003EWhile multimodal approaches to dermatological diagnostics are complex and require meticulous dataset curation and image acquisition, they have shown promise in enhancing performance and generalizability over unimodal methods\u003Csup\u003E\u003Ca id=\u0022xref-ref-8-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-8\u0022\u003E8\u003C\/a\u003E\u003C\/sup\u003E. Studies have demonstrated the efficacy of combining clinical and dermoscopic images to improve performance for both binary and multi-class classification tasks \u003Csup\u003E\u003Ca id=\u0022xref-ref-9-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-9\u0022\u003E9\u003C\/a\u003E,\u003Ca id=\u0022xref-ref-10-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-10\u0022\u003E10\u003C\/a\u003E\u003C\/sup\u003E. Incorporating clinical meta-data as an adjunct to image data has also been repeatedly explored, with mixed results in improving model performance \u003Csup\u003E\u003Ca id=\u0022xref-ref-11-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-11\u0022\u003E11\u003C\/a\u003E,\u003Ca id=\u0022xref-ref-12-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-12\u0022\u003E12\u003C\/a\u003E\u003C\/sup\u003E. Furthermore, these multimodal classifiers demonstrate better generalizability across datasets compared to their unimodal counterparts.\u003C\/p\u003E\u003Cp id=\u0022p-22\u0022\u003EAlthough multimodal strategies show great potential in improving diagnostic accuracy for skin conditions, their adoption is significantly limited by the scarcity of rigorous external benchmark evaluations and limited access to multimodal datasets. The barriers to widespread development and release of transparent, well-annotated multimodal datasets include increased burden to traditional clinical workflows, the laborious nature of annotating multiple images, and previously documented issues arising from institutional privacy policies and incentives that favor siloed datasets \u003Csup\u003E\u003Ca id=\u0022xref-ref-13-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-13\u0022\u003E13\u003C\/a\u003E\u003C\/sup\u003E. In addition, this lack of comprehensive validation poses an obstacle to the clinical utility of these algorithms. For instance, external validation of dermoscopic image-based algorithms in the International Skin Imaging Collaboration (ISIC) Grand Challenge revealed significant accuracy drops due to shifts in disease distributions, poor performance on conditions not represented in training, and routine clinical artifacts like hair or pen markings. Moreover, these algorithms often failed to recognize images beyond their training scope \u003Csup\u003E\u003Ca id=\u0022xref-ref-14-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-14\u0022\u003E14\u003C\/a\u003E\u003C\/sup\u003E. Similarly, our group previously showed that state-of-the-art (SOTA) clinical algorithms demonstrate marked performance declines on an independent retrospective benchmark database, especially for patients with Fitzpatrick V-VI skin tones \u003Csup\u003E\u003Ca id=\u0022xref-ref-15-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-15\u0022\u003E15\u003C\/a\u003E\u003C\/sup\u003E.\u003C\/p\u003E\u003Cp id=\u0022p-23\u0022\u003EPrior multimodal work has been aimed at pigmented lesion classification tasks, with retrospective curation limiting its applicability \u003Csup\u003E\u003Ca id=\u0022xref-ref-16-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-16\u0022\u003E16\u003C\/a\u003E,\u003Ca id=\u0022xref-ref-17-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-17\u0022\u003E17\u003C\/a\u003E\u003C\/sup\u003E The absence of \u003Cem\u003Eprospective\u003C\/em\u003E multimodal datasets for external validation across a variety of diagnoses hampers the AI field\u2019s ability to uncover and address algorithmic vulnerabilities or biases. Consequently, this limitation impedes the trust in the generalizability and clinical viability of dermatology-AI models.\u003C\/p\u003E\u003Cp id=\u0022p-24\u0022\u003ETo address this gap, we introduce the Melanoma Research Alliance Multimodal Image Dataset for AI-based Skin Cancer (MIDAS), the first publicly available, prospectively-recruited, systematically-paired dermoscopic and clinical image-based dataset across a range of skin-lesion diagnoses. This dual-center dataset encompasses a wide array of skin lesions and includes well-annotated, patient-level, clinical metadata. It aims to more accurately mirror real-world clinical scenarios than retrospectively curated datasets and is enhanced by extensive histopathologic confirmation to ensure data integrity. We also report an evaluation of four SOTA algorithms using the MIDAS dataset to assess their \u201cout-of-the-box\u201d performances. Additionally, we illustrate MIDAS\u2019s significance in multimodal dermatology diagnostics, addressing clinical implementation questions like optimal image distance for different models and laying the foundation for future multimodal AI development using MIDAS.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-2\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E2. Results\u003C\/h2\u003E\u003Cdiv id=\u0022sec-3\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E2.1 Demographics and dataset characteristics\u003C\/h3\u003E\u003Cp id=\u0022p-25\u0022\u003E796 total patients were prospectively enrolled in the study across two healthcare systems, with 734 from Stanford and 62 from Cleveland Clinic. Cleveland Clinic primarily participated as an additional expert pigmented lesion\/melanoma center, while the Stanford sites included predominantly general dermatology clinics along with an expert pigmented lesion\/melanoma clinic clinics. The population had a male predominance (64.9%), with a mean age of 60.9 years. The majority of patients represented clinician-assessed Fitzpatrick skin tones I\/II, with 16.8% representing Fitzpatrick III-VI skin tones (\u003Cstrong\u003E\u003Ca id=\u0022xref-table-wrap-1-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T1\u0022\u003ETable 1\u003C\/a\u003E\u003C\/strong\u003E).\u003C\/p\u003E\u003Cdiv id=\u0022T1\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/831014\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/831014\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/831014\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 1.\u003C\/span\u003E \u003Cspan class=\u0022caption-title\u0022\u003EDemographic characteristics of the cohort.\u003C\/span\u003E\u003Cp id=\u0022p-26\u0022 class=\u0022first-child\u0022\u003EPatients were recruited from both general dermatology and expert pigmented lesion\/melanoma clinics at Stanford. Cleveland Clinic also participated as an additional expert pigmented lesion\/melanoma center.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-27\u0022\u003EAmong all patients, we enrolled a total of 1290 unique lesions. This resulted in a total of 3830 images, which included 1187 dermoscopy images, 1198 clinical images taken from a distance of 15-cm, 1185 clinical images taken from 30-cm, and 260 virtual images from photos submitted by patients or their primary care physicians via the patient portal. Among these, 244 represented control lesions that were not biopsied, while 1046 (81.1%) were biopsied with histopathologic confirmation; the latter were subsequently used in the primary analysis for evaluation of model performance. We employed a total of 3162 images for primary analysis of model performances: 1048 dermoscopy images, 1056 clinical images taken from 15-cm away, 1045 clinical images taken from 30-cm away, and 13 from virtual encounters where lesions were ultimately biopsied.\u003C\/p\u003E\u003Cp id=\u0022p-28\u0022\u003EThe most common final histopathology-confirmed diagnosis was benign melanocytic nevus (22.4%), followed closely by basal cell carcinoma (20.3%) \u003Cstrong\u003E(\u003Ca id=\u0022xref-table-wrap-2-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T2\u0022\u003ETable 2\u003C\/a\u003E)\u003C\/strong\u003E. 135 total surgically eligible melanocytic lesions were diagnosed, including 46 (4.4%) invasive melanomas, 47 (4.5%) melanomas-in-situ, and 42 (4.0%) other surgically-eligible melanocytic diagnoses, such as severely dysplastic nevi and melanocytomas. In addition to melanocytic lesions, the dataset represents a relatively broad distribution of non-melanocytic lesions representing benign, pre-malignant, and malignant diagnoses commonly encountered in clinical practice, such as seborrheic keratoses (8.0%), actinic keratoses (6.0%), fibrous papules (0.6%), dermatofibromas (1.5%), and vascular lesions (1.0%).\u003C\/p\u003E\u003Cdiv id=\u0022T2\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/831010\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/831010\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/831010\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 2.\u003C\/span\u003E \u003Cspan class=\u0022caption-title\u0022\u003EBiopsied lesion characteristics.\u003C\/span\u003E\u003Cp id=\u0022p-29\u0022 class=\u0022first-child\u0022\u003EThese diagnoses were biopsy-proven and interpreted by 3 board-certified dermatopathologists at Stanford and 6 dermatopathologists at Cleveland Clinic. A dermatopathology consensus conference reviewed any diagnosis of severely dysplastic melanocytic nevus or worse at Stanford, and any diagnosis of invasive melanomas at CCF.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-30\u0022\u003EBefore releasing the dataset, we reviewed all images and performed additional preprocessing, which included cropping and further de-identification. Currently, all consented Stanford images are publicly released at [\u003Ca href=\u0022https:\/\/stanfordaimi.azurewebsites.net\/datasets\/f4c2020f-801a-42dd-a477-a1a8357ef2a5\u0022\u003Ehttps:\/\/stanfordaimi.azurewebsites.net\/datasets\/f4c2020f-801a-42dd-a477-a1a8357ef2a5\u003C\/a\u003E]\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-4\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E2.2 Clinician impressions\u003C\/h3\u003E\u003Cp id=\u0022p-31\u0022\u003EFor all biopsied lesions at Stanford, the initial treating dermatologist submitted their top 5 diagnoses based on their rank-ordered clinical impression prior to obtaining the biopsy results. We analyzed top 1 and top 3 diagnoses. Using top-1 diagnosis alone, clinicians were overall 79.3% accurate at differentiating a malignant from a benign lesion, with sensitivity and specificity of 78.4% and 80.0%, respectively \u003Cstrong\u003E(\u003Ca id=\u0022xref-fig-1-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F1\u0022\u003EFigure 1\u003C\/a\u003E)\u003C\/strong\u003E. This increased to 91.3% accuracy when evaluating top-3 benign vs. malignant diagnosis \u003Cstrong\u003E(Supplementary Table 1)\u003C\/strong\u003E. This dataset aims to be relevant for triage tasks and biopsy decision scenarios outside of an expert pigmented lesion clinic, where ruling in a concerning lesion takes primacy over ruling out a non-concerning lesion. Thus, we defined accuracy for clinicians around identifying the possibility of malignancy rather than providing the exact diagnosis.\u003C\/p\u003E\u003Cdiv id=\u0022F1\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F1.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022Confusion matrix for dermatologists\u0026#x2019; top-1 diagnosis. a) Confusion matrix is normalized by actual diagnosis to allow for better visualization of clinician performance. b) Absolute numbers are provided for this confusion matrix. Benign lesions include: AK, dermatofibroma, fibrous papule, hemangioma, SEMs, SK, melanocytic nevus. Malignant lesions include: BCC, melanoma, melanoma-in-situ, SCC, SCCIS.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-401821736\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;span xmlns=\u0026quot;http:\/\/www.w3.org\/1999\/xhtml\u0026quot; class=\u0026quot;caption-title\u0026quot;\u0026gt;Confusion matrix for dermatologists\u0026#x2019; top-1 diagnosis.\u0026lt;\/span\u0026gt; a) Confusion matrix is normalized by actual diagnosis to allow for better visualization of clinician performance. b) Absolute numbers are provided for this confusion matrix. Benign lesions include: AK, dermatofibroma, fibrous papule, hemangioma, SEMs, SK, melanocytic nevus. Malignant lesions include: BCC, melanoma, melanoma-in-situ, SCC, SCCIS.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 1.\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F1.medium.gif\u0022 width=\u0022440\u0022 height=\u0022282\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 1.\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F1.medium.gif\u0022 width=\u0022440\u0022 height=\u0022282\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F1.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 1.\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F1.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022 xmlns:xhtml=\u0022http:\/\/www.w3.org\/1999\/xhtml\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 1.\u003C\/span\u003E \u003Cspan class=\u0022caption-title\u0022\u003EConfusion matrix for dermatologists\u2019 top-1 diagnosis.\u003C\/span\u003E\u003Cp id=\u0022p-32\u0022 class=\u0022first-child\u0022\u003Ea) Confusion matrix is normalized by actual diagnosis to allow for better visualization of clinician performance. b) Absolute numbers are provided for this confusion matrix. Benign lesions include: AK, dermatofibroma, fibrous papule, hemangioma, SEMs, SK, melanocytic nevus. Malignant lesions include: BCC, melanoma, melanoma-in-situ, SCC, SCCIS.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-33\u0022\u003EFor the identification of melanomas (including invasive melanoma and melanoma-in-situ) for top-1 diagnosis, clinicians were 57% sensitive and 95% specific. When accounting for top-3 diagnoses, this increased to 94% sensitivity and 98% specificity. For top-3 diagnosis, clinicians were accurate if they mentioned the correct diagnosis in any of their top-3 differentials. We observed that most clinicians included both diagnoses of benign melanocytic neoplasms and malignant melanomas in their differentials for melanocytic lesions, making top-3 diagnosis less informative than top-1 diagnosis. Performance was high for the most common non-melanoma skin cancers, i.e., basal cell and squamous cell (keratinocyte) carcinomas. For basal cell carcinomas, using top-1 diagnosis, clinicians were 81% sensitive and 91% specific, while for invasive squamous cell carcinomas, clinicians were 65% sensitive and 97% specific using their top-1 diagnosis.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-5\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E2.3 Benchmark Model Performance\u003C\/h3\u003E\u003Cp id=\u0022p-34\u0022\u003EOur evaluation revealed a performance decline for all models compared to their initial metrics, although the extent and dimensions of the decline varied significantly according to the model (\u003Cstrong\u003E\u003Ca id=\u0022xref-fig-2-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F2\u0022\u003EFigure 2\u003C\/a\u003E\u003C\/strong\u003E, \u003Cstrong\u003E\u003Ca id=\u0022xref-table-wrap-3-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T3\u0022\u003ETable 3\u003C\/a\u003E\u003C\/strong\u003E).\u003C\/p\u003E\u003Cdiv id=\u0022F2\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-4.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022Model performance results-ROC curve, confusion matrix, and image examples. a) The evaluation metrics of the ADAE model, including the confusion matrix and ROC curve, alongside a random sample of the images that were correctly and incorrectly predicted are shown (far right). b) The performance of the HCI model is shown using the confusion matrix and multiclass ROC curve. The AUC for the \u0026#x2018;vasc\u0026#x2019; category is the highest at 0.94. Dermoscopic images of correctly predicted lesions are also shown in the far-right for all the seven disease categories. c) ModelDerm\u0026#x2019;s performance is equally shown and samples of true positive (TP), true negative (TN), false negative (FN), and false positive (FP) clinical images are displayed. d) This shows the results of the DeepDerm model in the form of nine-label and three-label confusion matrices. Random image samples that correspond to correct and incorrect predictions are also displayed in a way that correlates with the three-label matrix.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-401821736\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;span xmlns=\u0026quot;http:\/\/www.w3.org\/1999\/xhtml\u0026quot; class=\u0026quot;caption-title\u0026quot;\u0026gt;Model performance results-ROC curve, confusion matrix, and image examples.\u0026lt;\/span\u0026gt; a) The evaluation metrics of the ADAE model, including the confusion matrix and ROC curve, alongside a random sample of the images that were correctly and incorrectly predicted are shown (far right). b) The performance of the HCI model is shown using the confusion matrix and multiclass ROC curve. The AUC for the \u0026#x2018;vasc\u0026#x2019; category is the highest at 0.94. Dermoscopic images of correctly predicted lesions are also shown in the far-right for all the seven disease categories. c) ModelDerm\u0026#x2019;s performance is equally shown and samples of true positive (TP), true negative (TN), false negative (FN), and false positive (FP) clinical images are displayed. d) This shows the results of the DeepDerm model in the form of nine-label and three-label confusion matrices. Random image samples that correspond to correct and incorrect predictions are also displayed in a way that correlates with the three-label matrix.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 2.\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-4.medium.gif\u0022 width=\u0022440\u0022 height=\u0022363\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 2.\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-4.medium.gif\u0022 width=\u0022440\u0022 height=\u0022363\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-4.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download \u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-4.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-5.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022Model performance results-ROC curve, confusion matrix, and image examples. a) The evaluation metrics of the ADAE model, including the confusion matrix and ROC curve, alongside a random sample of the images that were correctly and incorrectly predicted are shown (far right). b) The performance of the HCI model is shown using the confusion matrix and multiclass ROC curve. The AUC for the \u0026#x2018;vasc\u0026#x2019; category is the highest at 0.94. Dermoscopic images of correctly predicted lesions are also shown in the far-right for all the seven disease categories. c) ModelDerm\u0026#x2019;s performance is equally shown and samples of true positive (TP), true negative (TN), false negative (FN), and false positive (FP) clinical images are displayed. d) This shows the results of the DeepDerm model in the form of nine-label and three-label confusion matrices. Random image samples that correspond to correct and incorrect predictions are also displayed in a way that correlates with the three-label matrix.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-401821736\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;span xmlns=\u0026quot;http:\/\/www.w3.org\/1999\/xhtml\u0026quot; class=\u0026quot;caption-title\u0026quot;\u0026gt;Model performance results-ROC curve, confusion matrix, and image examples.\u0026lt;\/span\u0026gt; a) The evaluation metrics of the ADAE model, including the confusion matrix and ROC curve, alongside a random sample of the images that were correctly and incorrectly predicted are shown (far right). b) The performance of the HCI model is shown using the confusion matrix and multiclass ROC curve. The AUC for the \u0026#x2018;vasc\u0026#x2019; category is the highest at 0.94. Dermoscopic images of correctly predicted lesions are also shown in the far-right for all the seven disease categories. c) ModelDerm\u0026#x2019;s performance is equally shown and samples of true positive (TP), true negative (TN), false negative (FN), and false positive (FP) clinical images are displayed. d) This shows the results of the DeepDerm model in the form of nine-label and three-label confusion matrices. Random image samples that correspond to correct and incorrect predictions are also displayed in a way that correlates with the three-label matrix.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 2.\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-5.medium.gif\u0022 width=\u0022440\u0022 height=\u0022326\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 2.\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-5.medium.gif\u0022 width=\u0022440\u0022 height=\u0022326\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-5.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download \u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/28\/2024.06.27.24309562\/F2\/graphic-5.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 2.\u003C\/span\u003E \u003Cspan class=\u0022caption-title\u0022\u003EModel performance results-ROC curve, confusion matrix, and image examples.\u003C\/span\u003E\u003Cp id=\u0022p-35\u0022 class=\u0022first-child\u0022\u003Ea) The evaluation metrics of the ADAE model, including the confusion matrix and ROC curve, alongside a random sample of the images that were correctly and incorrectly predicted are shown (far right). b) The performance of the HCI model is shown using the confusion matrix and multiclass ROC curve. The AUC for the \u2018vasc\u2019 category is the highest at 0.94. Dermoscopic images of correctly predicted lesions are also shown in the far-right for all the seven disease categories. c) ModelDerm\u2019s performance is equally shown and samples of true positive (TP), true negative (TN), false negative (FN), and false positive (FP) clinical images are displayed. d) This shows the results of the DeepDerm model in the form of nine-label and three-label confusion matrices. Random image samples that correspond to correct and incorrect predictions are also displayed in a way that correlates with the three-label matrix.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022T3\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/831013\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/831013\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/831013\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 3.\u003C\/span\u003E \u003Cspan class=\u0022caption-title\u0022\u003EModel Performance.\u003C\/span\u003E\u003Cp id=\u0022p-36\u0022 class=\u0022first-child\u0022\u003EThis depicts the performance of 4 state of the art models, which had varied inputs and use cases. We calculated the accuracy, sensitivity, specificity, precision and F1-score for each model.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-37\u0022\u003EThe \u003Cstrong\u003EADAE model\u003C\/strong\u003E, previously validated for melanoma diagnosis \u003Csup\u003E\u003Ca id=\u0022xref-ref-5-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-5\u0022\u003E5\u003C\/a\u003E\u003C\/sup\u003E, showed a slight AUC decrease to 0.82 (from 0.857), with high sensitivity of 96.9% (95% CI: 91.2\u201398.9%) that compared favorably with the original model sensitivity of 96.8% (95% CI:91.1-98.9%). However, it showed significantly lower specificity of 13.1% (95% CI: 11.1\u201315.4%) and accuracy of 21% (95% CI: 18.5-23.4%) on the MIDAS dataset \u003Cstrong\u003E(\u003Ca id=\u0022xref-table-wrap-3-2\u0022 class=\u0022xref-table\u0022 href=\u0022#T3\u0022\u003ETable 3\u003C\/a\u003E)\u003C\/strong\u003E, primarily due to increased false positives. Focusing only on MIDAS\u2019s melanocytic subset, ADAE\u2019s sensitivity remained high, with specificity and accuracy slightly improved to 17.0% (95% CI: 13.0\u201321.8%), and 37.5% (95% CI: 32.8\u201342.6%) respectively.\u003C\/p\u003E\u003Cp id=\u0022p-38\u0022\u003EAcross all disease categories, the \u003Cstrong\u003EHCI model\u003C\/strong\u003E achieved sensitivity of 43.3% (95% CI: 29.5-55.7%) and specificity of 90.4% (95% CI: 87.8-92.9%), with overall accuracy of 84.2% (95% CI: 80.1-88.9%), which was similar to the original results reported by Tschandl et al (80.3%). For malignant lesions, the model\u2019s sensitivity was 58.2% (95% CI: 56.9-65.2%), and specificity was 77.5% (95% CI: 74.1-81.9%), reflecting a drop in performance compared to its original study of 58.7% sensitivity and 93.3% specificity\u003Csup\u003E\u003Ca id=\u0022xref-ref-19-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-19\u0022\u003E19\u003C\/a\u003E\u003C\/sup\u003E.\u003C\/p\u003E\u003Cp id=\u0022p-39\u0022\u003E\u003Cstrong\u003EModelDerm\u2019s\u003C\/strong\u003E performance on MIDAS\u2019s distance of 15-cm and 30-cm clinical images revealed an AUC of 0.68 (95% CI: 0.66-0.71), which was notably lower than its retrospective validation results of 0.86 (95% CI: 0.85-0.88). The model\u2019s sensitivity and specificity for malignancy detection were 46.8% and 79.0%, respectively, indicating a notable variation from the original 62.7% (95% CI 59.9\u201365.1) sensitivity and 90.0% (95% CI 89.4\u201390.6) specificity \u003Csup\u003E\u003Ca id=\u0022xref-ref-24-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-24\u0022\u003E24\u003C\/a\u003E\u003C\/sup\u003E.\u003C\/p\u003E\u003Cp id=\u0022p-40\u0022\u003E\u003Cstrong\u003EDeepDerm\u2019s\u003C\/strong\u003E mean accuracy for the three-way classification task dropped to 43.3% (\u003Cstrong\u003E\u003Ca id=\u0022xref-table-wrap-3-3\u0022 class=\u0022xref-table\u0022 href=\u0022#T3\u0022\u003ETable 3\u003C\/a\u003E\u003C\/strong\u003E), which was a significant decrease from the initially reported 72.1% \u003Csup\u003E\u003Ca id=\u0022xref-ref-23-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-23\u0022\u003E23\u003C\/a\u003E\u003C\/sup\u003E. Its sensitivity for malignant diagnoses was low at 28.0%, with a specificity of 84.1%. Misclassification patterns revealed that malignant lesions were most commonly misclassified as non-neoplastic (inflammatory), followed by benign. We also explored performance discrepancies across the different hospital sites (\u003Cstrong\u003ESupplementary Table 2)\u003C\/strong\u003E.\u003C\/p\u003E\u003Cdiv id=\u0022sec-6\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E2.3.1 Poor performers\u003C\/h4\u003E\u003Cp id=\u0022p-41\u0022\u003EWithin this dataset, 27 lesions were identified as the most challenging cases, since all four models incorrectly classified every tested image at top-1 diagnosis. Clinicians demonstrated a 51.9% accuracy as top-1 diagnosis within this challenging subset of lesions. Basal cell carcinomas and squamous cell carcinoma in situ comprised the majority (20\/27) of these incorrectly classified images, but there was one melanoma within this dataset that was incorrectly predicted to represent a nevus by the clinicians as a top-1 diagnosis.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-7\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E2.3.2 Optimizing Image Acquisition Using Paired Camera Distances\u003C\/h4\u003E\u003Cp id=\u0022p-42\u0022\u003EThe optimal camera distance of image acquisition for algorithm performance was evaluated using both Deepderm and Modelderm, because both utilized clinical images either exclusively as one of the primary modalities in their training. When looking at the effects of camera distance on model performance, accuracy, and sensitivity improved for benign and malignant lesions with images acquired at 15-cm photography as compared to 30-cm photography across both models \u003Cstrong\u003E(\u003Ca id=\u0022xref-table-wrap-4-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T4\u0022\u003ETable 4\u003C\/a\u003E)\u003C\/strong\u003E. This was especially notable for DeepDerm in the classification of malignant lesions, with an improvement in sensitivity from 0.17 to 0.30. Overall accuracy for non-neoplastic disease classification by DeepDerm improved at 15-cm photography due to improved specificity but at a significant cost to sensitivity.\u003C\/p\u003E\u003Cdiv id=\u0022T4\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/831006\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/831006\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/831006\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 4.\u003C\/span\u003E \u003Cspan class=\u0022caption-title\u0022\u003EModel subanalysis: Image distance.\u003C\/span\u003E\u003Cp id=\u0022p-43\u0022 class=\u0022first-child\u0022\u003EWe explored the impact of image distance-dermoscopy, 15cm and 30cm on ModelDerm and DeepDerm performance. ModelDerm was only trained on clinical images; DeepDerm was trained on both dermoscopy images and clinical images.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-8\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E2.3.3 Image modality: Dermoscopy vs. clinical images\u003C\/h4\u003E\u003Cp id=\u0022p-44\u0022\u003EOf the four algorithms, DeepDerm was the only one that incorporated both dermoscopy images and clinical images in its training, and it thus provided the best vehicle to evaluate the comparative performance of dermoscopy and clinical images with a single algorithm \u003Csup\u003E\u003Ca id=\u0022xref-ref-23-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-23\u0022\u003E23\u003C\/a\u003E\u003C\/sup\u003E. As expected, we confirmed that ModelDerm performed worse on dermoscopy images compared to clinical images, since it was trained solely on clinical images \u003Cstrong\u003E(\u003Ca id=\u0022xref-table-wrap-4-2\u0022 class=\u0022xref-table\u0022 href=\u0022#T4\u0022\u003ETable 4\u003C\/a\u003E\u003C\/strong\u003E). In contrast, DeepDerm in general demonstrated its best performance on dermoscopy images, with notable improvement in its identification of non-neoplastic lesions. For the classification of malignant lesions, DeepDerm showed improvement in sensitivity from 0.30 to 0.37 moving from 15-cm clinical photography to dermoscopic images, with minimal impact on overall accuracy, suggesting a potential slight benefit for dermoscopy over clinical images for triage-based tasks where a higher sensitivity is favorable, though at the cost of slightly reduced specificity.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-9\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E2.3.4 Clinical characteristics influencing performance\u003C\/h4\u003E\u003Cp id=\u0022p-45\u0022\u003EWe explored the effects of demographics and other clinical information on model performance, including patient natal sex, age, prior history of melanoma, Fitzpatrick skin type, and lesion size \u003Cstrong\u003E(Supplementary Table 3)\u003C\/strong\u003E. We noted that ADAE performed better in patients with a prior history of melanoma (accuracy 28% vs 19% in those with no prior history), likely due to a higher prevalence of subsequent primary melanomas in patients with a history of melanoma. DeepDerm also performed better in lesions where patients had a prior history of melanoma (accuracy 73% vs 62% in patients with no prior history). We also found that ModelDerm performed better in lesions with Fitzpatrick skin type 3-6 (accuracy of 78% vs 59% in Fitzpatrick skin types 1-2). This improved performance of patients with darker skin types was maintained but with a smaller magnitude difference when we compared F1-scores, which are better evaluation metrics to compare subgroups with class imbalances (F1-score of 0.57 in Fitzpatrick 3-6 vs 0.53 in Fitzpatrick 1-2).\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-10\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E2.3.5 Ensemble models with multiple modalities\u003C\/h4\u003E\u003Cp id=\u0022p-46\u0022\u003EAs an exploratory analysis, we investigated the performance of multimodal evaluation utilizing both dermoscopic and clinical image input through ensembling of the outputs of two of the previously published SOTA algorithms (see Methods 4.4.3 Proof-of-Concept Ensemble Models). As shown in \u003Cstrong\u003E\u003Ca id=\u0022xref-table-wrap-5-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T5\u0022\u003ETable 5\u003C\/a\u003E\u003C\/strong\u003E, ensemble models A and B demonstrated increased accuracy at a cost of decreased sensitivity compared to the baseline models, ADAE and ModelDerm. Specifically, Ensemble Model A and B had accuracies of 75% compared to ADAE\u2019s baseline of 37% and ModelDerm\u2019s 74%. Notably, our logistic regression meta-model showed improved performance across multiple evaluation metrics with an AUC of 0.86 (95% CI: 0.76-0.94), accuracy of 82%, and specificity of 0.96.\u003C\/p\u003E\u003Cdiv id=\u0022T5\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/831007\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/831007\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/831007\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 5.\u003C\/span\u003E \u003Cspan class=\u0022caption-title\u0022\u003EEnsemble Model Performance on Melanocytic Lesions.\u003C\/span\u003E\u003Cp id=\u0022p-47\u0022 class=\u0022first-child\u0022\u003EWe explored model performance and ensemble models only on melanocytic lesions (melanocytic nevus, invasive melanoma, melanoma-in-situ, and surgically eligible melanocytic lesions). Ensemble Model A had weights of \u2153 ADAE dermoscopy + \u2153 ModelDerm 15cm clinical image + \u2153 ModelDerm 30cm clinical image. Ensemble Model B had weights of \u00bd ADAE dermoscopy + \u00bc ModelDerm 15cm clinical image + \u00bc ModelDerm 30cm clinical image. Ensemble Model C is a meta-model that employs logistic regression to learn the weights of the predictions from ADAE and ModelDerm.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-11\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E3. Discussion\u003C\/h2\u003E\u003Cp id=\u0022p-48\u0022\u003EThe translation of AI diagnostic tools into clinical practice, particularly in dermatology, presents a uniquely challenging and instructive paradigm due to the broad distribution of diagnoses and range of influencing factors such as skin tone, presence of artifacts, and variability in image acquisition. As these tools advance toward wider clinical deployment, the task of benchmarking and comprehensively understanding the strengths and vulnerabilities of SOTA diagnostic algorithms is increasingly paramount. The recent landmark U.S. \u201cExecutive Order on the Safe, Secure, and Trustworthy Development and Use of Artificial Intelligence\u201d emphasized the importance of assuring the quality of medical AI\u003Csup\u003E\u003Ca id=\u0022xref-ref-25-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-25\u0022\u003E25\u003C\/a\u003E\u003C\/sup\u003E. While the debate continues over the most valuable form of validation\u2014be it \u2018gold standard\u2019 external validation or local validation at the individual facility level \u003Csup\u003E\u003Ca id=\u0022xref-ref-26-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-26\u0022\u003E26\u003C\/a\u003E\u003C\/sup\u003E, it is clear that independent, pre-deployment assessments are an important way to evaluate the generalizability and reliability of these algorithms.\u003C\/p\u003E\u003Cp id=\u0022p-49\u0022\u003EStudies have shown that even within dermatologic practices, changing the intended clinical setting and distribution of diagnoses can significantly degrade algorithm performance, with the most striking performance reductions seen in more real-world, prospective contexts \u003Csup\u003E\u003Ca id=\u0022xref-ref-14-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-14\u0022\u003E14\u003C\/a\u003E,\u003Ca id=\u0022xref-ref-15-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-15\u0022\u003E15\u003C\/a\u003E,\u003Ca id=\u0022xref-ref-20-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-20\u0022\u003E20\u003C\/a\u003E\u003C\/sup\u003E. While these studies represent progress towards external validation for unimodal lesion classifiers, the lack of well-annotated, transparent multimodal datasets is a major impediment to the ultimate development of multimodal diagnostic tools. In this context, the MIDAS dataset enables the utility of multiple modalities, along with meta-data inputs for both pigmented and non-pigmented skin lesions, and it covers a breadth of benign, pre-malignant, and malignant lesion types.\u003C\/p\u003E\u003Cp id=\u0022p-50\u0022\u003EA key strength of our dataset is that it offers applicability in the evaluation of algorithms intended for diagnostic settings where a broader distribution of lesions is expected. These include general dermatology clinics, teledermatology-based practices, and primary care clinics. The prospective nature of MIDAS invariably included enrollment of lesions with secondary changes such as hemorrhagic crusts, excoriations, or edema from inflamed nevi or irritated seborrheic keratoses which have been shown to reduce algorithm performance \u003Csup\u003E\u003Ca id=\u0022xref-ref-14-3\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-14\u0022\u003E14\u003C\/a\u003E\u003C\/sup\u003E. This inclusion supports the evaluation of algorithms on these common, real-world image artifacts inherent to the diseases and patients. A limitation of the dataset is the lower representation of patients with Fitzpatrick IV-VI skin types. This likely resulted from both the lower incidence of skin cancer in people of color and potentially disparate access to care at academic medical centers. We anticipate a solution to this important issue is increasing the availability of transparent, publicly available datasets where skin cancer cases in patients of color can be increasingly aggregated, and we believe that reporting MIDAS\u2019s skin tone distribution and its public release can contribute to this effort. There are additional efforts in the community to increase knowledge about diagnostic imaging in skin of color\u003Csup\u003E\u003Ca id=\u0022xref-ref-27-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-27\u0022\u003E27\u003C\/a\u003E\u003C\/sup\u003E. In this respect, ModelDerm\u2019s outperformance of the other algorithms on patients with darker Fitzpatrick types may be due in part to its training data, which was generated in large part in South Korea, thus highlighting the potential impact of the demographics of patients represented within algorithms\u2019 training on their ultimate performance across diverse populations.\u003C\/p\u003E\u003Cp id=\u0022p-51\u0022\u003EIn our benchmarking experiments using multiple SOTA models utilizing a variety of inputs, we observed a decline in key performance measures, particularly sensitivity for malignant diagnoses and overall accuracy, across three of the four algorithms when deployed \u2019out of the box\u2019, without threshold adjustments or fine-tuning. This significant drop in model performance underscores the necessity for comprehensive pre-deployment analysis and local optimization prior to the clinical deployment of dermatologic classifiers. Moreover, it demonstrates how imperative ongoing performance monitoring is to ensure the model remains within the local control parameters, and highlights a potential regulatory challenge. Establishing regulatory frameworks that permit and govern such continuous monitoring will be crucial. From the algorithm development perspective, with the exception of ADAE, the algorithms were trained predominantly on data from a select number of institutions (typically one to three), hence broadening the institutional and geographic basis of future diagnostic algorithms may improve generalizability. One particularly striking pattern of errors revealed through this dataset was in the DeepDerm algorithm, where epidermal tumors were consistently miss-classified as non-neoplastic (inflammatory) lesions, suggesting the need for further fine-tuning to the task at hand. Finally, the development of MIDAS provides an instructive case in how performance can be negatively impacted when classifiers that were trained primarily on pigmented lesion classification are challenged with a broader disease distribution. This is of particular value for evaluating algorithms intended for practitioners assessing a wider variety of lesional diagnoses.\u003C\/p\u003E\u003Cp id=\u0022p-52\u0022\u003EOf note, both of the exclusively dermoscopic algorithms, HCI and ADAE, maintained reasonably robust AUCs in their respective classification tasks, implying significant potential benefit from pre-deployment optimization. ADAE, uniquely among the deployed algorithms, maintained a very high sensitivity for melanoma due to its design around a pre-specified 95% sensitivity threshold. However, with our broader dataset, it demonstrated a markedly reduced specificity of 13.1% (95% CI: 11.1\u201315.4%) which slightly increased to 17.0% (95% CI: 13.0\u201321.8%) in the subanalysis limited to melanocytic lesions. These results highlight the importance of understanding and adhering to the task for which an algorithm was originally trained, as ADAE was intended for use in cases where melanoma was in the differential diagnosis and thus provides post-clinician decision support to minimize unnecessary biopsies\/false positives rather than \u2018triaging in\u2019 potentially concerning lesions. Challenging the algorithm with non-melanocytic cases and other out-of-distribution clinical scenarios likely contributed to the observed change in performance characteristics.\u003C\/p\u003E\u003Cp id=\u0022p-53\u0022\u003EOur evaluation of ensemble approaches on MIDAS also yielded promising results, underscoring the dataset\u2019s utility in assessing multimodal strategies. These findings stress the importance of not only pre-deployment validation, but also post-deployment surveillance and regulatory aspects of maintaining algorithmic efficacy and safety in real-world intended settings. Our study is not without limitations, however, including restricted access to some models, which constrained our ability to explore fusion approaches and fully leverage the multimodal dataset for evaluation. In conclusion, this novel, publicly available MIDAS dataset can be utilized as a valuable tool in dermatology-AI, offering a rigorous benchmark for refining multimodal diagnostic models and addressing the complexities of real-world clinical applications.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-12\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E4. Methods\u003C\/h2\u003E\u003Cdiv id=\u0022sec-13\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E4.1 Patient enrollment and image acquisition\u003C\/h3\u003E\u003Cp id=\u0022p-54\u0022\u003EAll research was approved by the Institutional Review Board at Stanford University under IRB#36050, along with the Cleveland Clinic Foundation under IRB#20-666, and adhered to the Helsinki Declaration. Patients presenting to the dermatology clinics of participating dermatologists at Stanford Medicine or Cleveland Clinic Foundation between August 18, 2020, and April 17, 2023, were eligible for the study if 1) they had at least one solitary skin lesion of concern identified where a skin biopsy was deemed medically necessary by the dermatologist investigator or 2) patients were directed to in-clinic evaluation for a lesion that was previously identified as concerning through a teledermatology encounter or dermatologist review of a patient photo submitted through the electronic patient messaging portal. Patients underwent written informed consent with either the physician or research coordinator, after which both clinical and dermoscopic digital photography were obtained of any eligible skin lesions. Each lesion underwent standardized photography with a contemporary model iPhone or iPad device (iPhone SE to iPhone 12 Pro and iPod touch to iPad mini) without flash photography at 15-cm and 30-cm distances, along with digital dermatoscope photography. For each lesion, clinical information about the patient was obtained and recorded including sex assigned at birth, age, Fitzpatrick skin type, personal history of melanoma, anatomic location, and the lesion\u2019s length and width. Investigators had the discretion to identify additional control lesions that clinically appeared benign on a corresponding contralateral body site that were similarly enrolled for digital photography as an un-biopsied control lesion to include in the dataset, though primary analysis was restricted to biopsied lesions.\u003C\/p\u003E\u003Cp id=\u0022p-55\u0022\u003ENine Stanford dermatologists, two Cleveland Clinic dermatologists, and one Cleveland Clinic surgical oncologist recruited patients primarily from adult general medical dermatology clinics (Stanford) along with a subset (12.6%) from specialized pigmented lesion and melanoma clinics (Stanford and Cleveland). At the time of first enrollment, the four Stanford dermatologists at the specialized pigmented lesion and melanoma clinic had an average of 15.7 years of post-residency experience, while those in general medical dermatology clinics had an average of 3.9 years\u2019 experience. Cleveland Clinic dermatologists\/surgical oncologists had an average of 25.5 years of post-residency experience.\u003C\/p\u003E\u003Cp id=\u0022p-56\u0022\u003EDermatologists noted their top five ranked clinical impressions at the time of evaluation, along with their binary level of confidence (Yes\/No) in their top impression.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-14\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E4.2 Reference labeling\u003C\/h3\u003E\u003Cp id=\u0022p-57\u0022\u003EFor any biopsied lesions, associated histopathologic final diagnoses were recorded and categorized into a previously described taxonomy (ZC, AC, RN). Biopsy results were interpreted by three board-certified dermatopathologists at Stanford and 6 dermatopathologists at Cleveland Clinic. A dermatopathology consensus conference reviewed any diagnosis of severely dysplastic melanocytic nevus or worse at Stanford, and any diagnosis of invasive melanomas at CCF. Melanocytic lesions were specifically grouped in the following manner: benign melanocytic nevi, melanomas (including melanoma in-situ and invasive melanoma), and surgically-eligible intermediate melanocytic tumors where complete excision is typically recommended at Stanford and Cleveland Clinic (including severely dysplastic melanocytic nevi and melanocytomas such as typical\/atypical Spitz tumors, BAP-1-inactivated melanocytic tumors, deep penetrating nevi\/tumors, and cellular blue nevi with atypia). For lesions deemed as controls or not biopsied, the top favored diagnosis by the original treating physician was chosen as the reference label. Cases were included in the dataset if a second reviewing independent board-certified dermatologist (ZC) agreed with the favored diagnosis based on a review of the associated images.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-15\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E4.3 Data preprocessing and exploratory analysis\u003C\/h3\u003E\u003Cp id=\u0022p-58\u0022\u003EFor all images collected, we focused on the subset of lesions that were biopsied with a pathology report. We cleaned the dataset by ensuring consistency across demographic labeling and correcting any missing data in the metadata. We manually parsed through all images in the dataset to ensure image quality. To better understand our dataset, we conducted exploratory analyses on patient demographics and lesion diagnostic breakdown. To maintain consistency across our analysis, for melanocytic lesions, we categorized invasive melanoma and melanoma-in-situ as malignant lesions. All other melanocytic lesions, including surgically-eligible melanocytic lesions, were categorized as benign lesions. In contrast, the SOTA algorithms (see section below for algorithm details) had different classifications of actinic keratosis; ADAE and ModelDerm categorized these lesions as benign, while the HCI model and DeepDerm classified these as malignant. In our analysis, we categorized actinic keratoses (AKs) as clinically benign, but for the HCI and DeepDerm models we kept the original classification and noted them as malignant. This difference in classification yields a unique perspective where the categorization of a lesion can be influenced by model selection - an academic task of differentiating between malignant and malignant-potential lesions, versus a triage-based task of identifying which lesions may need to be biopsied to rule out malignancies.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-16\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E4.4 Evaluating models\u003C\/h3\u003E\u003Cp id=\u0022p-59\u0022\u003EGiven our dataset\u2019s prospectively-recruited and multimodal nature, we explored its potential use as a benchmark tool for external validation and comparative analysis of algorithms that leverage different elements of MIDAS\u2019s image and clinical metadata. Specifically, we examined the performance of four previously published SOTA models, each employing distinct input modalities as described in subsequent sections, to understand their performance on this prospective data. Our analysis included two models based on dermoscopy images, one model predicated on clinical images, and one model designed to process either clinical or dermoscopic images.\u003C\/p\u003E\u003Cp id=\u0022p-60\u0022\u003ETo assess their \u2019out of the box\u2019 performance, we adhered to the threshold values as stipulated by the original models. All four models were tested on the entire biopsy-proven analysis dataset. However, the evaluation of each model was tailored to the primary classification task for which it was initially designed, such as the binary classification of melanoma vs. non-melanoma, or malignant vs. benign lesions, and multi-class categorization.\u003C\/p\u003E\u003Cdiv id=\u0022sec-17\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E4.4.1 Dermoscopy-based Models\u003C\/h4\u003E\u003Cdiv id=\u0022sec-18\u0022 class=\u0022subsection\u0022\u003E\u003Ch5\u003EAll Data are External (ADAE)\u003C\/h5\u003E\u003Cp id=\u0022p-61\u0022\u003EThe ADAE algorithm is an open-source, non-commercial dermoscopic model that emerged as the top-ranking algorithm in the ISIC 2020 melanoma classification challenge \u003Csup\u003E\u003Ca id=\u0022xref-ref-5-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-5\u0022\u003E5\u003C\/a\u003E\u003C\/sup\u003E. This ensemble model comprises 18 separate CNN-based models, which were trained on 5-folds totaling 90 models, each contributing to a collective probability class label. Notably, four model architectures incorporate metadata fields (age, sex, anatomic location, width, and height of images) to enhance prediction accuracy. Initially designed as a melanoma screening tool, the ADAE was thresholded to achieve high sensitivity (above 95%) for melanoma detection. We focused on a version of the model recently detailed by Marchetti et al.\u003Csup\u003E\u003Ca id=\u0022xref-ref-5-3\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-5\u0022\u003E5\u003C\/a\u003E\u003C\/sup\u003E, which demonstrated an AUC of 0.857 and sensitivity of 96.8% (95% CI: 91.1-98.9%) in a prospective validation study.\u003C\/p\u003E\u003Cp id=\u0022p-62\u0022\u003EOur evaluation involved 1046 dermoscopic images from our dataset, examining the model\u2019s proficiency in identifying melanoma, defined inclusively as invasive melanoma and melanoma-in-situ. Additionally, we performed a focused sub-analysis on melanocytic lesions to closely align with the ADAE model\u2019s primary use case, where melanoma is considered a key differential diagnosis. Consistent with its primary training objective, a prediction by the modified ADAE was deemed accurate if the corresponding true label was either melanoma-in-situ or invasive melanoma, aligning with the model\u2019s intended triage role.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-19\u0022 class=\u0022subsection\u0022\u003E\u003Ch5\u003EHCI (Human\u2013Computer Interaction) Model\u003C\/h5\u003E\u003Cp id=\u0022p-63\u0022\u003EThe HCI model, developed by Tschandl et. al, is a ResNet34-based architecture that was fine-tuned for multi-class classification on the HAM10000 pigmented lesions dataset \u003Csup\u003E\u003Ca id=\u0022xref-ref-18-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-18\u0022\u003E18\u003C\/a\u003E\u003C\/sup\u003E. The original model outputs probabilities for seven categories: actinic keratoses and intraepithelial carcinoma\/Bowen disease (akiec), basal cell carcinoma (bcc), benign lesions of the keratosis type including solar lentigines\/seborrheic keratoses and lichen-planus like keratosis (bkl), dermatofibroma (df), melanoma (mel), melanocytic nevi (nv) and vascular lesions, including angiomas, angiokeratomas, pyogenic granulomas, and hemorrhages (vasc)\u003Csup\u003E\u003Ca id=\u0022xref-ref-19-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-19\u0022\u003E19\u003C\/a\u003E\u003C\/sup\u003E. In this study, we evaluated the model on our prospectively-collected dermoscopic dataset.\u003C\/p\u003E\u003Cp id=\u0022p-64\u0022\u003EImages of rare diseases not represented by the HCI model\u2019s predefined categories were excluded to maintain category consistency. All pathologic diagnoses were mapped to the seven-category framework and discrepancies were resolved by two board-certified dermatologists (R.N. and R.D.). Following this mapping process, we ran the model in test mode on 935 dermoscopic images: nv: 276, bcc: 212, akiec: 190, bkl: 124, mel: 97, df: 22, vasc: 14. Both category-specific and overall performance metrics like AUC, sensitivity, and accuracy were calculated. Additionally, specific metrics for malignant cases were reported, providing a comprehensive overview of the model\u2019s capabilities.\u003C\/p\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-20\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E4.4.2 Clinical Image and Clinical Image + Dermoscopy-based Models\u003C\/h4\u003E\u003Cdiv id=\u0022sec-21\u0022 class=\u0022subsection\u0022\u003E\u003Ch5\u003EModelDerm\u003C\/h5\u003E\u003Cp id=\u0022p-65\u0022\u003EModelDerm, an ensemble of ResNet variants that was trained on clinical images of skin lesions, can classify 184 skin diseases \u003Csup\u003E\u003Ca id=\u0022xref-ref-20-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-20\u0022\u003E20\u003C\/a\u003E,\u003Ca id=\u0022xref-ref-21-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-21\u0022\u003E21\u003C\/a\u003E\u003C\/sup\u003E. Similar to ADAE and HCI models, we based our methodology on Han et al. \u003Csup\u003E\u003Ca id=\u0022xref-ref-20-3\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-20\u0022\u003E20\u003C\/a\u003E,\u003Ca id=\u0022xref-ref-21-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-21\u0022\u003E21\u003C\/a\u003E\u003C\/sup\u003E, using the same thresholds as defined by their study. We used the 2019 build version of the model. The malignancy score was calculated as the sum of malignant outputs and 0.2[\u00d7[premalignant outputs as previously defined\u003Csup\u003E\u003Ca id=\u0022xref-ref-22-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-22\u0022\u003E22\u003C\/a\u003E\u003C\/sup\u003E. The algorithm consists of two modules: a lesion detector followed by the ResNet ensemble classifier. We excluded 3% (98\/3162) of images where ModelDerm was unable to detect a lesion.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-22\u0022 class=\u0022subsection\u0022\u003E\u003Ch5\u003EDeepDerm\u003C\/h5\u003E\u003Cp id=\u0022p-66\u0022\u003EThe DeepDerm model, developed similarly to Esteva et al.\u2019s algorithm, was trained on a comprehensive dataset including images from the ISIC Dermoscopic Archive, the Edinburgh Dermofit Library, and Stanford Medicine \u003Csup\u003E\u003Ca id=\u0022xref-ref-23-3\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-23\u0022\u003E23\u003C\/a\u003E\u003C\/sup\u003E. Employing the ResNet50 architecture, the model was trained to classify skin conditions into a detailed 836-category taxonomy, which also simplifies into nine and three-category frameworks representing various dermatological conditions \u003Csup\u003E\u003Ca id=\u0022xref-ref-23-4\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-23\u0022\u003E23\u003C\/a\u003E\u003C\/sup\u003E.\u003C\/p\u003E\u003Cp id=\u0022p-67\u0022\u003EHerein, we aligned pathology reports from our prospectively collected dataset with the DeepDerm taxonomy, resolving discrepancies through a consensus approach among two board-certified dermatologists (R.N. and R.D.). The model\u2019s performance, including category-specific and overall metrics, was evaluated on the dataset, alongside examining its performance across the two participating centers.\u003C\/p\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-23\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E4.4.3 Proof-of-Concept Ensemble Models\u003C\/h4\u003E\u003Cp id=\u0022p-68\u0022\u003EWe further investigated the potential use of the MRA-MIDAS dataset for the development of multimodal ensemble approaches. We developed a proof-of-concept weight-based ensemble model using ADAE and ModelDerm, focusing on dermoscopy and clinical images. Due to the proprietary nature of the models, our ensemble approach was limited to the final output layers, employing two strategies: 1) a weighted average model and 2) a logistic regression-based meta-model. The weighted average model assigned: a) equal weights, distributing \u2153 each to ADAE dermoscopy, ModelDerm 15-cm and 30-cm predictions. b) higher weight to ADAE (\u00bd) with the remaining \u00bd equally divided between ModelDerm 15-cm and 30-cm predictions.\u003C\/p\u003E\u003Cp id=\u0022p-69\u0022\u003EFor the meta-model, we utilized a logistic regression algorithm to integrate predictions from ADAE and ModelDerm (15-cm and 30-cm). It was optimized via grid search over various regularization parameters, and we employed five-fold cross-validation for training. The evaluation metrics were based on a held-out test set. For consistent comparison in our ensemble models, our analyses were conducted on melanocytic lesions (melanocytic nevus, invasive melanoma, melanoma-in-situ, and surgically eligible melanocytic lesions).\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-24\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E4.4.4 Statistical Analysis\u003C\/h4\u003E\u003Cp id=\u0022p-70\u0022\u003EConfidence intervals were calculated using the bootstrapping method and all analyses were performed using Python (version 3.8), and Scikit-learn (version 1.2).\u003C\/p\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-25\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EConflicts of Interest\u003C\/h2\u003E\u003Cp id=\u0022p-71\u0022\u003EAC was an investigator for Skin Analytics. JK is consultant to Hims, Enspectra Health, research collaborator with Google Research, and on advisory board for Skin Analytics. PT received honoraria from Silverchair, unrestricted grants for education projects from Lilly, and honoraria for lectures from AbbVie, Lilly, FotoFinder and Novartis. SSH is the founder, chief executive officer, and chief technical officer of IDerma, Inc. RD has served as an advisor to MDAlgorithms and Revea and received consulting fees from Pfizer, L\u2019Oreal, Frazier Healthcare Partners, and DWA, and research funding from Union Chimique Belge (UCB), and was an investigator for Skin Analytics. RN is a consultant to Enspectra Health and Sanctum, LLC and was an investigator for Skin Analytics.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-26\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EFunding\u003C\/h2\u003E\u003Cp id=\u0022p-72\u0022\u003EThis publication is based on research supported by the Melanoma Research Alliance (MRA)-L\u2019Oreal Dermatological Beauty Brands Team Science Award, along with philanthropic funding from the David Mair and Vanessa Vu-Mair Artificial Intelligence in Skin Cancer Fund and the Tal \u0026amp; Cinthia Simon Melanoma Research Fund at Stanford Medicine. We also received a Stanford Human-Centered Artificial Intelligence Google Cloud Credits Grant to support compute efforts.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-27\u0022\u003E\u003Ch2 class=\u0022\u0022\u003ESUPPLEMENTARY\u003C\/h2\u003E\u003Cdiv id=\u0022T6\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/831009\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/831009\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/831009\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ESupplementary Table 1.\u003C\/span\u003E \u003Cp id=\u0022p-74\u0022 class=\u0022first-child\u0022\u003EClinician Impressions-Top1 and Top3.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022T7\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/831008\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/831008\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/831008\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ESupplementary Table 2.\u003C\/span\u003E \u003Cp id=\u0022p-75\u0022 class=\u0022first-child\u0022\u003EModel subanalysis-Stanford vs Cleveland Clinic.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022T8\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/831015\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/831015\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/831015\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ESupplementary Table 3.\u003C\/span\u003E \u003Cspan class=\u0022caption-title\u0022\u003EModel subanalysis: Demographic Subgroups\u003C\/span\u003E\u003Cp id=\u0022p-76\u0022 class=\u0022first-child\u0022\u003ELarge lesions were defined as length and width of the lesions are above the cut-off value of 6mm. Small lesions are all the other lesions.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section data-availability\u0022 id=\u0022sec-28\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EData Availability\u003C\/h2\u003E\u003Cp id=\u0022p-77\u0022\u003EData produced are available online at \u003Ca href=\u0022https:\/\/stanfordaimi.azurewebsites.net\/datasets\/f4c2020f-801a-42dd-a477-a1a8357ef2a5\u0022\u003Ehttps:\/\/stanfordaimi.azurewebsites.net\/datasets\/f4c2020f-801a-42dd-a477-a1a8357ef2a5\u003C\/a\u003E\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section ack\u0022 id=\u0022ack-1\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EAcknowledgments\u003C\/h2\u003E\u003Cp id=\u0022p-73\u0022\u003EThis material is the result of work supported with resources and the use of facilities at the Veterans Affairs Palo Alto Health Care System in Palo Alto, California. We would like to acknowledge Vijaytha Muralidharan, Susan Weber, and other team members who have helped support this work.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section fn-group\u0022 id=\u0022fn-group-1\u0022\u003E\u003Ch2\u003EFootnotes\u003C\/h2\u003E\u003Cul\u003E\u003Cli class=\u0022fn-other\u0022 id=\u0022fn-1\u0022\u003E\u003Cp id=\u0022p-1\u0022\u003E\u003Ca class=\u0022rev-xref\u0022 href=\u0022#xref-fn-1-1\u0022\u003E\u21b5\u003C\/a\u003E\u003Cspan class=\u0022fn-label\u0022\u003E*\u003C\/span\u003E These authors share lead authorship\u003C\/p\u003E\u003C\/li\u003E\u003Cli class=\u0022fn-other\u0022 id=\u0022fn-2\u0022\u003E\u003Cp id=\u0022p-2\u0022\u003E\u003Ca class=\u0022rev-xref\u0022 href=\u0022#xref-fn-2-1\u0022\u003E\u21b5\u003C\/a\u003E\u003Cspan class=\u0022fn-label\u0022\u003E**\u003C\/span\u003E These authors share co-senior authorship\u003C\/p\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section ref-list\u0022 id=\u0022ref-list-1\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EReferences\u003C\/h2\u003E\u003Col class=\u0022cit-list ref-use-labels\u0022\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E1.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-1-1\u0022 title=\u0022View reference 1. in text\u0022 id=\u0022ref-1\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.1\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ECoustasse\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EA.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ESarkar\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ER.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EAbodunde\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EB.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EMetzger\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EB. J.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ESlater\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EC. M\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003EUse of Teledermatology to Improve Dermatological Access in Rural Areas\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ETelemed. J. E. Health\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E25\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E1022\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E1032\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2019\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DTelemed.%2BJ.%2BE.%2BHealth%26rft.volume%253D25%26rft.spage%253D1022%26rft_id%253Dinfo%253Apmid%252Fhttp%253A%252F%252Fwww.n%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=http:\/\/www.n\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F28%2F2024.06.27.24309562.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E2.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-2-1\u0022 title=\u0022View reference 2. in text\u0022 id=\u0022ref-2\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.2\u0022 data-doi=\u002210.1016\/S1470-2045(02)00679-4\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EKittler\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EH.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EPehamberger\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EH.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EWolff\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EK.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EBinder\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003EDiagnostic accuracy of dermoscopy\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ELancet Oncol\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E3\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E159\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E165\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2002\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DThe%2Blancet%2Boncology%26rft.stitle%253DLancet%2BOncol%26rft.aulast%253DKittler%26rft.auinit1%253DH.%26rft.volume%253D3%26rft.issue%253D3%26rft.spage%253D159%26rft.epage%253D165%26rft.atitle%253DDiagnostic%2Baccuracy%2Bof%2Bdermoscopy.%26rft_id%253Dinfo%253Adoi%252F10.1016%252FS1470-2045%252802%252900679-4%26rft_id%253Dinfo%253Apmid%252F11902502%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1016\/S1470-2045(02)00679-4\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=11902502\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F28%2F2024.06.27.24309562.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=000174890300020\u0026amp;link_type=ISI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-newisilink cit-ref-sprinkles-webofscience\u0022\u003E\u003Cspan\u003EWeb of Science\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E3.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-3-1\u0022 title=\u0022View reference 3. in text\u0022 id=\u0022ref-3\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-book\u0022 id=\u0022cit-2024.06.27.24309562v1.3\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EMcKay\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-source\u0022\u003EThe Dermatologic History\u003C\/span\u003E. (\u003Cspan class=\u0022cit-publ-name\u0022\u003EButterworths\u003C\/span\u003E, \u003Cspan class=\u0022cit-pub-date\u0022\u003E1990\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E4.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-4-1\u0022 title=\u0022View reference 4. in text\u0022 id=\u0022ref-4\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.4\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EMu\u00f1oz-L\u00f3pez\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EC.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EPerformance of a deep neural network in teledermatology: a single-centre prospective diagnostic study\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJ. Eur. Acad. Dermatol. Venereol\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E35\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E546\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E553\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2021\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJ.%2BEur.%2BAcad.%2BDermatol.%2BVenereol%26rft.volume%253D35%26rft.spage%253D546%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E5.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-5-1\u0022 title=\u0022View reference 5. in text\u0022 id=\u0022ref-5\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.5\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EMarchetti\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM. A.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EProspective validation of dermoscopy-based open-source artificial intelligence for melanoma diagnosis (PROVE-AI study)\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ENPJ Digit Med\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E6\u003C\/span\u003E, \u003Cspan class=\u0022cit-issue\u0022\u003E127\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E6.\u003C\/span\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal no-rev-xref\u0022 id=\u0022cit-2024.06.27.24309562v1.6\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EPhillips\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EAssessment of Accuracy of an Artificial Intelligence Algorithm to Detect Melanoma in Images of Skin Lesions\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJAMA Netw Open\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E2\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003Ee1913436\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2019\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJAMA%2BNetw%2BOpen%26rft.volume%253D2%26rft.spage%253D1913436e%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E7.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-7-1\u0022 title=\u0022View reference 7. in text\u0022 id=\u0022ref-7\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.7\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EMenzies\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ES. W.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EComparison of humans versus mobile phone-powered artificial intelligence for the diagnosis and management of pigmented skin cancer in secondary care: a multicentre, prospective, diagnostic, clinical trial\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ELancet Digit Health\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E5\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003Ee679\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003Ee691\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DLancet%2BDigit%2BHealth%26rft.volume%253D5%26rft.spage%253D679e%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E8.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-8-1\u0022 title=\u0022View reference 8. in text\u0022 id=\u0022ref-8\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.8\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ELuo\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EN.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EArtificial intelligence-assisted dermatology diagnosis: From unimodal to multimodal\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EComput. Biol. Med\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E165\u003C\/span\u003E, \u003Cspan class=\u0022cit-issue\u0022\u003E107413\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E9.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-9-1\u0022 title=\u0022View reference 9. in text\u0022 id=\u0022ref-9\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.9\u0022 data-doi=\u002210.1111\/exd.13777\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EYap\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJ.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EYolland\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EW.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ETschandl\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EP\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003EMultimodal skin lesion classification using deep learning\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EExp. Dermatol\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E27\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E1261\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E1267\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2018\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DExp.%2BDermatol%26rft.volume%253D27%26rft.spage%253D1261%26rft_id%253Dinfo%253Adoi%252F10.1111%252Fexd.13777%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1111\/exd.13777\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E10.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-10-1\u0022 title=\u0022View reference 10. in text\u0022 id=\u0022ref-10\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.10\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EWang\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EY.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ECai\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJ.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ELouie\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ED. C.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EWang\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EZ. J.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ELee\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ET. K\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003EIncorporating clinical knowledge with constrained classifier chain into a multimodal deep network for melanoma detection\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EComput. Biol. Med\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E137\u003C\/span\u003E, \u003Cspan class=\u0022cit-issue\u0022\u003E104812\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2021\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E11.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-11-1\u0022 title=\u0022View reference 11. in text\u0022 id=\u0022ref-11\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.11\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EOu\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EC.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EA deep learning based multimodal fusion model for skin lesion diagnosis using smartphone collected clinical images and metadata\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EFront Surg\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E9\u003C\/span\u003E, \u003Cspan class=\u0022cit-issue\u0022\u003E1029991\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2022\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E12.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-12-1\u0022 title=\u0022View reference 12. in text\u0022 id=\u0022ref-12\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.12\u0022 data-doi=\u002210.1109\/ISBI45749.2020.9098645\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ELi\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EW.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EZhuang\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJ.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EWang\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ER.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EZhang\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJ.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EZheng\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EW.-S.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EFusing Metadata and Dermoscopy Images for Skin Disease Diagnosis\u003C\/span\u003E. in 2020 \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EIEEE 17th International Symposium on Biomedical Imaging (ISBI)\u003C\/abbr\u003E \u003Cspan class=\u0022cit-fpage\u0022\u003E1996\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E2000\u003C\/span\u003E (IEEE, \u003Cspan class=\u0022cit-pub-date\u0022\u003E2020\u003C\/span\u003E).\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id-scheme\u0022\u003Edoi:\u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E10.1109\/ISBI45749.2020.9098645\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DIEEE%2B17th%2BInternational%2BSymposium%2Bon%2BBiomedical%2BImaging%2B%2528ISBI%2529%26rft_id%253Dinfo%253Adoi%252F10.1109%252FISBI45749.2020.9098645%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1109\/ISBI45749.2020.9098645\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E13.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-13-1\u0022 title=\u0022View reference 13. in text\u0022 id=\u0022ref-13\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.13\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EDaneshjou\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ER.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ESmith\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM. P.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ESun\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM. D.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ERotemberg\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EV.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EZou\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJ\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003ELack of Transparency and Potential Bias in Artificial Intelligence Data Sets and Algorithms: A Scoping Review\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJAMA Dermatol\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E157\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E1362\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E1369\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2021\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJAMA%2BDermatol%26rft.volume%253D157%26rft.spage%253D1362%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E14.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-14-1\u0022 title=\u0022View reference 14. in text\u0022 id=\u0022ref-14\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.14\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ECombalia\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EValidation of artificial intelligence prediction models for skin cancer diagnosis using dermoscopy images: the 2019 International Skin Imaging Collaboration Grand Challenge\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ELancet Digit Health\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E4\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003Ee330\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003Ee339\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2022\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DLancet%2BDigit%2BHealth%26rft.volume%253D4%26rft.spage%253De330%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E15.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-15-1\u0022 title=\u0022View reference 15. in text\u0022 id=\u0022ref-15\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.15\u0022 data-doi=\u002210.1126\/sciadv.abq6147\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EDaneshjou\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ER.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EDisparities in dermatology AI performance on a diverse, curated clinical image set\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ESci Adv\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E8\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003Eeabq6147\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2022\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DSci%2BAdv%26rft.volume%253D8%26rft.spage%253Deabq6147%26rft_id%253Dinfo%253Adoi%252F10.1126%252Fsciadv.abq6147%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1126\/sciadv.abq6147\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E16.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-16-1\u0022 title=\u0022View reference 16. in text\u0022 id=\u0022ref-16\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.16\u0022 data-doi=\u002210.1109\/JBHI.2018.2824327\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EKawahara\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJ.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EDaneshvar\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ES.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EArgenziano\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EG.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EHamarneh\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EG\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003E7-Point Checklist and Skin Lesion Classification using Multi-Task Multi-Modal Neural Nets\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EIEEE J Biomed Health Inform\u003C\/abbr\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2018\u003C\/span\u003E)\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id-scheme\u0022\u003Edoi:\u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E10.1109\/JBHI.2018.2824327\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DIEEE%2BJ%2BBiomed%2BHealth%2BInform%26rft.volume%253D23%26rft.spage%253D538%26rft_id%253Dinfo%253Adoi%252F10.1109%252FJBHI.2018.2824327%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1109\/JBHI.2018.2824327\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E17.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-17-1\u0022 title=\u0022View reference 17. in text\u0022 id=\u0022ref-17\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.17\u0022 data-doi=\u002210.1016\/S2589-7500(21)00252-1\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EWen\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ED.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003ECharacteristics of publicly available skin cancer image datasets: a systematic review\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ELancet Digit Health\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E4\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003Ee64\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003Ee74\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2022\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DLancet%2BDigit%2BHealth%26rft.volume%253D4%26rft.spage%253D64e%26rft_id%253Dinfo%253Adoi%252F10.1016%252FS2589-7500%252821%252900252-1%26rft_id%253Dinfo%253Apmid%252F34772649%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1016\/S2589-7500(21)00252-1\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=34772649\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F28%2F2024.06.27.24309562.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E18.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-18-1\u0022 title=\u0022View reference 18. in text\u0022 id=\u0022ref-18\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.18\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ETschandl\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EP.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ERosendahl\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EC.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EKittler\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EH\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003EThe HAM10000 dataset, a large collection of multi-source dermatoscopic images of common pigmented skin lesions\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ESci Data\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E5\u003C\/span\u003E, \u003Cspan class=\u0022cit-issue\u0022\u003E180161\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2018\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E19.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-19-1\u0022 title=\u0022View reference 19. in text\u0022 id=\u0022ref-19\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.19\u0022 data-doi=\u002210.1038\/s41591-020-0942-0\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ETschandl\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EP.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EHuman-computer collaboration for skin cancer recognition\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ENat. Med\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E26\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E1229\u003C\/span\u003E\u2013 \u003Cspan class=\u0022cit-lpage\u0022\u003E1234\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2020\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DNat.%2BMed%26rft.volume%253D26%26rft.spage%253D1229%26rft_id%253Dinfo%253Adoi%252F10.1038%252Fs41591-020-0942-0%26rft_id%253Dinfo%253Apmid%252Fhttp%253A%252F%252Fwww.n%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1038\/s41591-020-0942-0\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=http:\/\/www.n\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F28%2F2024.06.27.24309562.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E20.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-20-1\u0022 title=\u0022View reference 20. in text\u0022 id=\u0022ref-20\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.20\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EHan\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ES. S.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EThe degradation of performance of a state-of-the-art skin image classifier when applied to patient-driven internet search\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ESci. Rep\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E12\u003C\/span\u003E, \u003Cspan class=\u0022cit-issue\u0022\u003E16260\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2022\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E21.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-21-1\u0022 title=\u0022View reference 21. in text\u0022 id=\u0022ref-21\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.21\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EHan\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ES. S.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EClassification of the Clinical Images for Benign and Malignant Cutaneous Tumors Using a Deep Learning Algorithm\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJ. Invest. Dermatol\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E138\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E1529\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E1538\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2018\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJ.%2BInvest.%2BDermatol%26rft.volume%253D138%26rft.spage%253D1529%26rft_id%253Dinfo%253Apmid%252Fhttp%253A%252F%252Fwww.n%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=http:\/\/www.n\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F28%2F2024.06.27.24309562.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E22.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-22-1\u0022 title=\u0022View reference 22. in text\u0022 id=\u0022ref-22\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.22\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EHan\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ES. S.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EKeratinocytic Skin Cancer Detection on the Face Using Region-Based Convolutional Neural Network\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJAMA Dermatol\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E156\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E29\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E37\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2020\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJAMA%2BDermatol%26rft.volume%253D156%26rft.spage%253D29%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E23.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-23-1\u0022 title=\u0022View reference 23. in text\u0022 id=\u0022ref-23\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.23\u0022 data-doi=\u002210.1038\/nature21056\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EEsteva\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EA.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EDermatologist-level classification of skin cancer with deep neural networks\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ENature\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E542\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E115\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E118\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2017\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DNature%26rft.volume%253D542%26rft.spage%253D115%26rft_id%253Dinfo%253Adoi%252F10.1038%252Fnature21056%26rft_id%253Dinfo%253Apmid%252F28117445%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1038\/nature21056\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=28117445\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F28%2F2024.06.27.24309562.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E24.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-24-1\u0022 title=\u0022View reference 24. in text\u0022 id=\u0022ref-24\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.24\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EHan\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ES. S.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EAssessment of deep neural networks for the diagnosis of benign and malignant skin neoplasms in comparison with dermatologists: A retrospective validation study\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EPLoS Med\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E17\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003Ee1003381\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2020\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DPLoS%2BMed%26rft.volume%253D17%26rft.spage%253De1003381%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E25.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-25-1\u0022 title=\u0022View reference 25. in text\u0022 id=\u0022ref-25\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.25\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EMello\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM. M.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EShah\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EN. H.\u003C\/span\u003E\u003C\/span\u003E \u0026amp; \u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EChar\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ED. S\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003EPresident Biden\u2019s Executive Order on Artificial Intelligence-Implications for Health Care Organizations\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJAMA\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E331\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E17\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E18\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2024\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJAMA%26rft.volume%253D331%26rft.spage%253D17%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E26.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-26-1\u0022 title=\u0022View reference 26. in text\u0022 id=\u0022ref-26\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.27.24309562v1.26\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EYoussef\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EA.\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-article-title\u0022\u003EExternal validation of AI models in health should be replaced with recurring local validation\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ENat. Med\u003C\/abbr\u003E. \u003Cspan class=\u0022cit-vol\u0022\u003E29\u003C\/span\u003E, \u003Cspan class=\u0022cit-fpage\u0022\u003E2686\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E2687\u003C\/span\u003E (\u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E).\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DNat.%2BMed%26rft.volume%253D29%26rft.spage%253D2686%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label\u0022\u003E27.\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-27-1\u0022 title=\u0022View reference 27. in text\u0022 id=\u0022ref-27\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-web\u0022 id=\u0022cit-2024.06.27.24309562v1.27\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-article-title\u0022\u003E2023 vol 13, S1| dermoscopy in skin of color: The journey so far\u003C\/span\u003E. \u003Ca href=\u0022https:\/\/dpcj.org\/index.php\/dpc\/issue\/view\/68\u0022\u003Ehttps:\/\/dpcj.org\/index.php\/dpc\/issue\/view\/68\u003C\/a\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003C\/ol\u003E\u003C\/div\u003E\u003Cspan class=\u0022highwire-journal-article-marker-end\u0022\u003E\u003C\/span\u003E\u003C\/div\u003E\u003Cspan class=\u0022related-urls\u0022\u003E\u003C\/span\u003E\u003C\/div\u003E\u003C\/div\u003E  \u003C\/div\u003E\n\n  \n  \u003C\/div\u003E\n\u003C\/div\u003E\n  \u003C\/div\u003E\n\u003C\/div\u003E\n\u003C\/div\u003E\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/js\/js_zP7WWIfzbyzvaM63L39cNV2juU_1XVH7wduFK9gcMNI.js\u0022\u003E\u003C\/script\u003E\n\u003C\/body\u003E\u003C\/html\u003E"}