{"markup":"\u003C?xml version=\u00221.0\u0022 encoding=\u0022UTF-8\u0022 ?\u003E\n    \u003Chtml version=\u0022HTML+RDFa+MathML 1.1\u0022\n    xmlns:content=\u0022http:\/\/purl.org\/rss\/1.0\/modules\/content\/\u0022\n    xmlns:dc=\u0022http:\/\/purl.org\/dc\/terms\/\u0022\n    xmlns:foaf=\u0022http:\/\/xmlns.com\/foaf\/0.1\/\u0022\n    xmlns:og=\u0022http:\/\/ogp.me\/ns#\u0022\n    xmlns:rdfs=\u0022http:\/\/www.w3.org\/2000\/01\/rdf-schema#\u0022\n    xmlns:sioc=\u0022http:\/\/rdfs.org\/sioc\/ns#\u0022\n    xmlns:sioct=\u0022http:\/\/rdfs.org\/sioc\/types#\u0022\n    xmlns:skos=\u0022http:\/\/www.w3.org\/2004\/02\/skos\/core#\u0022\n    xmlns:xsd=\u0022http:\/\/www.w3.org\/2001\/XMLSchema#\u0022\n    xmlns:mml=\u0022http:\/\/www.w3.org\/1998\/Math\/MathML\u0022\u003E\n  \u003Chead\u003E\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022\/\/cdn.jsdelivr.net\/qtip2\/2.2.1\/jquery.qtip.min.js\u0022\u003E\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/js\/js_YjAJQgxDlFX6S-O02jj9jCrVbrwlY3CGgCg1FzPlvBs.js\u0022\u003E\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022\u003E\n\u003C!--\/\/--\u003E\u003C![CDATA[\/\/\u003E\u003C!--\nif(typeof window.MathJax === \u0022undefined\u0022) window.MathJax = { menuSettings: { zoom: \u0022Click\u0022 } };\n\/\/--\u003E\u003C!]]\u003E\n\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/js\/js_waP91NpgGpectm_6Y2XDEauLJ8WCSCBKmmA87unpp2E.js\u0022\u003E\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022https:\/\/www.googletagmanager.com\/gtag\/js?id=G-0K57TCX5BY\u0022\u003E\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022\u003E\n\u003C!--\/\/--\u003E\u003C![CDATA[\/\/\u003E\u003C!--\nwindow.dataLayer = window.dataLayer || [];function gtag(){dataLayer.push(arguments)};gtag(\u0022js\u0022, new Date());gtag(\u0022set\u0022, \u0022developer_id.dMDhkMT\u0022, true);gtag(\u0022config\u0022, \u0022G-0K57TCX5BY\u0022, {\u0022groups\u0022:\u0022default\u0022,\u0022anonymize_ip\u0022:true});\n\/\/--\u003E\u003C!]]\u003E\n\u003C\/script\u003E\n\u003Cscript type=\u0022text\/javascript\u0022\u003E\n\u003C!--\/\/--\u003E\u003C![CDATA[\/\/\u003E\u003C!--\njQuery.extend(Drupal.settings, {\u0022basePath\u0022:\u0022\\\/\u0022,\u0022pathPrefix\u0022:\u0022\u0022,\u0022highwire\u0022:{\u0022ac\u0022:{\u0022medrxiv;2024.06.24.24309416v1\u0022:{\u0022access\u0022:{\u0022full\u0022:true},\u0022pisa_id\u0022:\u0022medrxiv;2024.06.24.24309416v1\u0022,\u0022apath\u0022:\u0022\u0022,\u0022jcode\u0022:\u0022medrxiv\u0022}},\u0022processed\u0022:[\u0022highwire_math\u0022],\u0022markup\u0022:[{\u0022requested\u0022:\u0022full-text\u0022,\u0022variant\u0022:\u0022full-text\u0022,\u0022view\u0022:\u0022full\u0022,\u0022pisa\u0022:\u0022medrxiv;2024.06.24.24309416v1\u0022}]},\u0022instances\u0022:\u0022{\\u0022highwire_abstract_tooltip\\u0022:{\\u0022content\\u0022:{\\u0022text\\u0022:\\u0022\\u0022},\\u0022style\\u0022:{\\u0022tip\\u0022:{\\u0022width\\u0022:20,\\u0022height\\u0022:20,\\u0022border\\u0022:1,\\u0022offset\\u0022:0,\\u0022corner\\u0022:true},\\u0022classes\\u0022:\\u0022qtip-custom hw-tooltip hw-abstract-tooltip qtip-shadow qtip-rounded\\u0022,\\u0022classes_custom\\u0022:\\u0022hw-tooltip hw-abstract-tooltip\\u0022},\\u0022position\\u0022:{\\u0022at\\u0022:\\u0022right center\\u0022,\\u0022my\\u0022:\\u0022left center\\u0022,\\u0022viewport\\u0022:true,\\u0022adjust\\u0022:{\\u0022method\\u0022:\\u0022shift\\u0022}},\\u0022show\\u0022:{\\u0022event\\u0022:\\u0022mouseenter click \\u0022,\\u0022solo\\u0022:true},\\u0022hide\\u0022:{\\u0022event\\u0022:\\u0022mouseleave \\u0022,\\u0022fixed\\u0022:1,\\u0022delay\\u0022:\\u0022100\\u0022}},\\u0022highwire_author_tooltip\\u0022:{\\u0022content\\u0022:{\\u0022text\\u0022:\\u0022\\u0022},\\u0022style\\u0022:{\\u0022tip\\u0022:{\\u0022width\\u0022:15,\\u0022height\\u0022:15,\\u0022border\\u0022:1,\\u0022offset\\u0022:0,\\u0022corner\\u0022:true},\\u0022classes\\u0022:\\u0022qtip-custom hw-tooltip hw-author-tooltip qtip-shadow qtip-rounded\\u0022,\\u0022classes_custom\\u0022:\\u0022hw-tooltip hw-author-tooltip\\u0022},\\u0022position\\u0022:{\\u0022at\\u0022:\\u0022top center\\u0022,\\u0022my\\u0022:\\u0022bottom center\\u0022,\\u0022viewport\\u0022:true,\\u0022adjust\\u0022:{\\u0022method\\u0022:\\u0022\\u0022}},\\u0022show\\u0022:{\\u0022event\\u0022:\\u0022mouseenter \\u0022,\\u0022solo\\u0022:true},\\u0022hide\\u0022:{\\u0022event\\u0022:\\u0022mouseleave \\u0022,\\u0022fixed\\u0022:1,\\u0022delay\\u0022:\\u0022100\\u0022}},\\u0022highwire_reflinks_tooltip\\u0022:{\\u0022content\\u0022:{\\u0022text\\u0022:\\u0022\\u0022},\\u0022style\\u0022:{\\u0022tip\\u0022:{\\u0022width\\u0022:15,\\u0022height\\u0022:15,\\u0022border\\u0022:1,\\u0022mimic\\u0022:\\u0022top center\\u0022,\\u0022offset\\u0022:0,\\u0022corner\\u0022:true},\\u0022classes\\u0022:\\u0022qtip-custom hw-tooltip hw-ref-link-tooltip qtip-shadow qtip-rounded\\u0022,\\u0022classes_custom\\u0022:\\u0022hw-tooltip hw-ref-link-tooltip\\u0022},\\u0022position\\u0022:{\\u0022at\\u0022:\\u0022bottom left\\u0022,\\u0022my\\u0022:\\u0022top left\\u0022,\\u0022viewport\\u0022:true,\\u0022adjust\\u0022:{\\u0022method\\u0022:\\u0022flip\\u0022}},\\u0022show\\u0022:{\\u0022event\\u0022:\\u0022mouseenter \\u0022,\\u0022solo\\u0022:true},\\u0022hide\\u0022:{\\u0022event\\u0022:\\u0022mouseleave \\u0022,\\u0022fixed\\u0022:1,\\u0022delay\\u0022:\\u0022100\\u0022}}}\u0022,\u0022qtipDebug\u0022:\u0022{\\u0022leaveElement\\u0022:0}\u0022,\u0022googleanalytics\u0022:{\u0022account\u0022:[\u0022G-0K57TCX5BY\u0022],\u0022trackOutbound\u0022:1,\u0022trackMailto\u0022:1,\u0022trackDownload\u0022:1,\u0022trackDownloadExtensions\u0022:\u00227z|aac|arc|arj|asf|asx|avi|bin|csv|doc(x|m)?|dot(x|m)?|exe|flv|gif|gz|gzip|hqx|jar|jpe?g|js|mp(2|3|4|e?g)|mov(ie)?|msi|msp|pdf|phps|png|ppt(x|m)?|pot(x|m)?|pps(x|m)?|ppam|sld(x|m)?|thmx|qtm?|ra(m|r)?|sea|sit|tar|tgz|torrent|txt|wav|wma|wmv|wpd|xls(x|m|b)?|xlt(x|m)|xlam|xml|z|zip\u0022,\u0022trackColorbox\u0022:1},\u0022ajaxPageState\u0022:{\u0022js\u0022:{\u0022\\\/\\\/cdn.jsdelivr.net\\\/qtip2\\\/2.2.1\\\/jquery.qtip.min.js\u0022:1,\u0022sites\\\/all\\\/modules\\\/highwire\\\/highwire\\\/plugins\\\/highwire_markup_process\\\/js\\\/highwire_article_reference_popup.js\u0022:1,\u0022sites\\\/all\\\/modules\\\/highwire\\\/highwire\\\/plugins\\\/highwire_markup_process\\\/js\\\/highwire_at_symbol.js\u0022:1,\u00220\u0022:1,\u0022sites\\\/all\\\/modules\\\/contrib\\\/google_analytics\\\/googleanalytics.js\u0022:1,\u0022https:\\\/\\\/www.googletagmanager.com\\\/gtag\\\/js?id=G-0K57TCX5BY\u0022:1,\u00221\u0022:1}}});\n\/\/--\u003E\u003C!]]\u003E\n\u003C\/script\u003E\n\u003Clink type=\u0022text\/css\u0022 rel=\u0022stylesheet\u0022 href=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/advagg_css\/css__dn-cpI1YtkU_iLHgA5WhlkxgYWyat_IxjF_B-WSYrpE__a9hIbt0eaZ7d5nhwnm2weG8R_2eXK4EvoOx9dOxouHE__QrrGUc7CpljPR5Aph-ukPbcwtK4AWrHGwCEXJ_k1V_c.css\u0022 media=\u0022all\u0022 \/\u003E\n\u003Clink type=\u0022text\/css\u0022 rel=\u0022stylesheet\u0022 href=\u0022\/\/cdn.jsdelivr.net\/qtip2\/2.2.1\/jquery.qtip.min.css\u0022 media=\u0022all\u0022 \/\u003E\n\u003Clink type=\u0022text\/css\u0022 rel=\u0022stylesheet\u0022 href=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/advagg_css\/css__HGACIFBlu2o05y3afvqlt5wrE_5Dn6MXsexfuEpeIwg__t4SOPxucAPoV3Os7g8dXqyMB1HRXQridRJ82X7nE33E__QrrGUc7CpljPR5Aph-ukPbcwtK4AWrHGwCEXJ_k1V_c.css\u0022 media=\u0022all\u0022 \/\u003E\n\u003Clink rel=\u0027stylesheet\u0027 type=\u0027text\/css\u0027 href=\u0027\/sites\/all\/modules\/contrib\/panels\/plugins\/layouts\/onecol\/onecol.css\u0027 \/\u003E\u003C\/head\u003E\u003Cbody\u003E\u003Cdiv class=\u0022panels-ajax-tab-panel panels-ajax-tab-panel-article-tab-full-text\u0022\u003E\u003Cdiv class=\u0022panel-display panel-1col clearfix\u0022 \u003E\n  \u003Cdiv class=\u0022panel-panel panel-col\u0022\u003E\n    \u003Cdiv\u003E\u003Cdiv class=\u0022panel-pane pane-highwire-markup\u0022 \u003E\n  \n      \n  \n  \u003Cdiv class=\u0022pane-content\u0022\u003E\n    \u003Cdiv class=\u0022highwire-markup\u0022\u003E\u003Cdiv xmlns=\u0022http:\/\/www.w3.org\/1999\/xhtml\u0022 data-highwire-cite-ref-tooltip-instance=\u0022highwire_reflinks_tooltip\u0022 class=\u0022content-block-markup\u0022 xmlns:xhtml=\u0022http:\/\/www.w3.org\/1999\/xhtml\u0022\u003E\u003Cdiv class=\u0022article fulltext-view \u0022\u003E\u003Cspan class=\u0022highwire-journal-article-marker-start\u0022\u003E\u003C\/span\u003E\u003Cdiv class=\u0022section abstract\u0022 id=\u0022abstract-1\u0022\u003E\u003Ch2\u003EAbstract\u003C\/h2\u003E\u003Cp id=\u0022p-2\u0022\u003ECombining predictions from multiple models into an ensemble is a widely used practice across many fields with demonstrated performance benefits. The R package \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E provides a flexible framework for ensembling various types of predictions, including point estimates and probabilistic predictions. A range of common methods for generating ensembles are supported, including weighted averages, quantile averages, and linear pools. The \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package fits within a broader framework of open-source software and data tools called the \u201chubverse\u201d, which facilitates the development and management of collaborative modelling exercises.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-1\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E1 Introduction\u003C\/h2\u003E\u003Cp id=\u0022p-13\u0022\u003EPredictions of future outcomes are essential to planning and decision making, yet generating reliable predictions of the future is challenging. One method for overcoming this challenge is combining predictions across multiple, independent models. These combination methods (also called aggregation or ensembling) have been repeatedly shown to produce predictions that are more accurate (\u003Ca id=\u0022xref-ref-7-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-7\u0022\u003EClemen 1989\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-28-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-28\u0022\u003ETimmermann 2006\u003C\/a\u003E) and more consistent (\u003Ca id=\u0022xref-ref-12-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-12\u0022\u003EHibon and Evgeniou 2005\u003C\/a\u003E) than individual models. Because of the clear performance benefits, multimodel ensembles are commonplace across fields, including weather (\u003Ca id=\u0022xref-ref-2-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-2\u0022\u003EAlley, Emanuel, and Zhang 2019\u003C\/a\u003E), climate (\u003Ca id=\u0022xref-ref-27-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-27\u0022\u003ETebaldi and Knutti 2007\u003C\/a\u003E), and economics (\u003Ca id=\u0022xref-ref-1-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-1\u0022\u003EAastveit et al. 2018\u003C\/a\u003E). More recently, multi-model ensembles have been used to improve predictions of infectious disease outbreaks (\u003Ca id=\u0022xref-ref-29-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-29\u0022\u003EViboud et al. 2018\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-14-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-14\u0022\u003EJohansson et al. 2019\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-16-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-16\u0022\u003EMcGowan et al. 2019\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-25-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-25\u0022\u003EReich et al. 2019\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-11-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-11\u0022\u003ECramer et al. 2022\u003C\/a\u003E).\u003C\/p\u003E\u003Cp id=\u0022p-14\u0022\u003EIn the rapidly growing field of outbreak forecasting, there are many proposed methods for generating ensembles. Generally, these methods differ in at least one of two ways: (1) the function used to combine or \u201caverage\u201d predictions, and (2) how predictions are weighted when performing the combination. No one method is universally \u201cthe best\u201d; a simple average of predictions works surprisingly well across a range of settings (\u003Ca id=\u0022xref-ref-16-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-16\u0022\u003EMcGowan et al. 2019\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-18-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-18\u0022\u003EPaireau et al. 2022\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-20-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-20\u0022\u003ERay et al. 2023\u003C\/a\u003E) for established theoretical reasons (\u003Ca id=\u0022xref-ref-32-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-32\u0022\u003EWinkler 2015\u003C\/a\u003E). However, more complex approaches have also been shown to have benefits in some settings (\u003Ca id=\u0022xref-ref-33-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-33\u0022\u003EYamana, Kandula, and Shaman 2016\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-22-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-22\u0022\u003ERay and Reich 2018\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-25-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-25\u0022\u003EReich et al. 2019\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-8-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-8\u0022\u003ECol\u00f3n-Gonz\u00e1lez et al. 2021\u003C\/a\u003E). Here, we present the \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package, which provides a flexible framework for generating ensemble predictions from multiple models. Complementing other software for combining predictions from multiple models (e.g., \u003Ca id=\u0022xref-ref-19-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-19\u0022\u003EPedregosa et al. (2011)\u003C\/a\u003E; Weiss, Raviv, and Roetzer (2019); \u003Ca id=\u0022xref-ref-4-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-4\u0022\u003EBosse et al. (2023)\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-10-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-10\u0022\u003ECouch and Kuhn (2023)\u003C\/a\u003E), \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E supports multiple types of predictions, including point estimates and different kinds of probabilistic predictions. Throughout, we will use the term \u201cprediction\u201d to refer to any kind of model output that may be combined including a forecast, a scenario projection, or a parameter estimate.\u003C\/p\u003E\u003Cp id=\u0022p-15\u0022\u003EThe \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package is part of the \u201chubverse\u201d collection of open-source software and data tools. The hubverse project facilitates the development and management of collaborative modelling exercises (\u003Ca id=\u0022xref-ref-9-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-9\u0022\u003EConsortium of Infectious Disease Modeling Hubs 2024\u003C\/a\u003E). The broader hub-verse initiative is motivated by the demonstrated benefits of collaborative hubs (\u003Ca id=\u0022xref-ref-24-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-24\u0022\u003EReich et al. 2022\u003C\/a\u003E; \u003Ca id=\u0022xref-ref-3-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-3\u0022\u003EBorchering et al. 2023\u003C\/a\u003E), including performance benefits of multi-model ensembles and the desire for standardization across such hubs. In this paper, we focus specifically on the functionality encompassed in \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E. We provide an overview of the methods implemented, including mathematical definitions and properties (\u003Ca id=\u0022xref-sec-2-1\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-2\u0022\u003ESection 2\u003C\/a\u003E) as well as implementation details (\u003Ca id=\u0022xref-sec-3-1\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-3\u0022\u003ESection 3\u003C\/a\u003E), a basic demonstration of functionality with simple examples (\u003Ca id=\u0022xref-sec-8-1\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-8\u0022\u003ESection 4\u003C\/a\u003E), and a more in-depth analysis using real influenza forecasts (\u003Ca id=\u0022xref-sec-14-1\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-14\u0022\u003ESection 5\u003C\/a\u003E) that motivates a discussion and comparison of the various methods (\u003Ca id=\u0022xref-sec-17-1\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-17\u0022\u003ESection 6\u003C\/a\u003E).\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-2\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E2 Mathematical definitions and properties of ensemble methods\u003C\/h2\u003E\u003Cp id=\u0022p-16\u0022\u003EThe \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package supports both point predictions and probabilistic predictions of different formats. A point prediction gives a single estimate of a future outcome while a probabilistic prediction provides an estimated probability distribution over a set of future outcomes. We use \u003Cem\u003EN\u003C\/em\u003E to denote the total number of individual predictions that the ensemble will combine. For example, these predictions will often be produced by different statistical or mathematical models, and \u003Cem\u003EN\u003C\/em\u003E is the total number of models that have provided predictions. Individual predictions will be indexed by the subscript \u003Cem\u003Ei\u003C\/em\u003E. Optionally, the package allows for calculating ensembles that use a weight \u003Cem\u003Ew\u003C\/em\u003E\u003Csub\u003E\u003Cem\u003Ei\u003C\/em\u003E\u003C\/sub\u003Efor each prediction; we define the set of model-specific weights as \u003Cstrong\u003E\u003Cem\u003Ew\u003C\/em\u003E\u003C\/strong\u003E = {\u003Cem\u003Ew\u003C\/em\u003E\u003Csub\u003E\u003Cem\u003Ei\u003C\/em\u003E\u003C\/sub\u003E|\u003Cem\u003Ei\u003C\/em\u003E \u2208 1, \u2026, \u003Cem\u003EN\u003C\/em\u003E}. Informally, predictions with a larger weight have a greater influence on the ensemble prediction, though the details of this depend on the ensemble method (described further below).\u003C\/p\u003E\u003Cp id=\u0022p-17\u0022\u003EFor a set of \u003Cem\u003EN\u003C\/em\u003E point predictions, \u003Cstrong\u003E\u003Cem\u003Ep\u003C\/em\u003E\u003C\/strong\u003E = {\u003Cem\u003Ep\u003C\/em\u003E\u003Csub\u003E\u003Cem\u003Ei\u003C\/em\u003E\u003C\/sub\u003E|\u003Cem\u003Ei\u003C\/em\u003E \u2208 1, \u2026, \u003Cem\u003EN\u003C\/em\u003E}, each from a distinct model \u003Cem\u003Ei\u003C\/em\u003E, the \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package can compute an ensemble of these predictions\n\u003Cspan class=\u0022disp-formula\u0022 id=\u0022disp-formula-1\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/graphic-1.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/graphic-1.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\n\n\u003C\/span\u003E\nusing any function \u003Cem\u003EC\u003C\/em\u003E and any set of model-specific weights \u003Cem\u003Ew\u003C\/em\u003E. For example, an arithmetic average of predictions yields \u003Cspan class=\u0022inline-formula\u0022 id=\u0022inline-formula-1\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-1.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-1.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/span\u003E, where the weights are non-negative and sum to 1. If \u003Cem\u003Ew\u003C\/em\u003E\u003Csub\u003E\u003Cem\u003Ei\u003C\/em\u003E\u003C\/sub\u003E = 1\/\u003Cem\u003EN\u003C\/em\u003E for all \u003Cem\u003Ei\u003C\/em\u003E, all predictions will be equally weighted. This framework can also support more complex functions for aggregation, such as a (weighted) median or geometric mean.\u003C\/p\u003E\u003Cp id=\u0022p-18\u0022\u003EFor probabilistic predictions, there are two commonly used classes of methods to average or ensemble multiple predictions: quantile averaging (also called a Vincent average (\u003Ca id=\u0022xref-ref-30-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-30\u0022\u003EVincent 1912\u003C\/a\u003E)) and probability averaging (also called a distributional mixture or linear opinion pool (\u003Ca id=\u0022xref-ref-26-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-26\u0022\u003EStone 1961\u003C\/a\u003E)) (\u003Ca id=\u0022xref-ref-15-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-15\u0022\u003ELichtendahl, Grushka-Cockayne, and Winkler 2013\u003C\/a\u003E). To define these two classes of methods, let \u003Cem\u003EF\u003C\/em\u003E (\u003Cem\u003Ex\u003C\/em\u003E) be a cumulative density function (CDF) defined over values \u003Cem\u003Ex\u003C\/em\u003E of the target variable for the prediction, and \u003Cem\u003EF\u003C\/em\u003E \u003Csup\u003E\u22121\u003C\/sup\u003E(\u003Cem\u003E\u03b8\u003C\/em\u003E) be the corresponding quantile function defined over quantile levels \u003Cem\u003E\u03b8\u003C\/em\u003E \u2208 [0, 1]. Throughout this article, we may refer to \u003Cem\u003Ex\u003C\/em\u003E as either a \u2018value of the target variable\u2019 or a \u2018quantile\u2019 depending on the context, and similarly we may refer to \u003Cem\u003E\u03b8\u003C\/em\u003E as either a \u2018quantile level\u2019 or a \u2018(cumulative) probability\u2019. Additionally, we will use \u003Cem\u003Ef\u003C\/em\u003E(\u003Cem\u003Ex\u003C\/em\u003E) to denote a probability mass function (PMF) for a prediction of a discrete variable or a discretization (such as binned values) of a continuous variable.\u003C\/p\u003E\u003Cp id=\u0022p-19\u0022\u003EThe quantile average combines a set of quantile functions, \u003Cspan class=\u0022inline-formula\u0022 id=\u0022inline-formula-2\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-2.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-2.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/span\u003E, with a given set of weights, \u003Cem\u003Ew\u003C\/em\u003E, as\n\u003Cspan class=\u0022disp-formula\u0022 id=\u0022disp-formula-2\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/graphic-2.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/graphic-2.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\n\n\u003C\/span\u003E\nThis computes the average value of predictions across different models for each fixed quantile level \u003Cem\u003E\u03b8\u003C\/em\u003E. For a normal distribution or any distribution with a shape and scale parameter, the resulting quantile average will be the same distribution, and the shape and scale parameters will be the average of the shape and scale parameters from the individual distributions (\u003Ca id=\u0022xref-fig-1-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F1\u0022\u003EFigure 1\u003C\/a\u003E, panel B). In other words, this method inteprets the predictive probability distributions that are being combined as uncertain estimates of a single true distribution. It is also possible to use other combination functions, such as a weighted median, to combine quantile predictions.\u003C\/p\u003E\u003Cdiv id=\u0022F1\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F1.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022(Panel A) Example of quantile output type predictions. In this example, points show model output collected for seven fixed quantile levels (\u0026#x3B8; = 0.01, 0.1, 0.3, 0.5, 0.7, 0.9, and 0.99) from two distributions (N(100, 10) in purple and N(120, 5) in green), with the underlying cumulative distribution functions (CDFs) shown with curves. The y-axis ticks show each of the fixed quantile levels. The associated values for each fixed quantile level do not align across distributions (vertical lines). (Panel B) Quantile average ensemble, which is calculated by averaging values for each fixed quantile level (represented by horizontal dashed gray lines). The distributions and corresponding model outputs from panel A are re-plotted and the black line shows the resulting quantile average ensemble. Inset shows corresponding probability density functions (PDFs). (Panel C) Linear pool ensemble, which is calculated by averaging cumulative probabilities for each fixed value (represented by vertical dashed gray lines). The distributions and corresponding model outputs from panel A are replotted. To calculate the linear pool in this case, where model outputs are not defined for the same values, the model outputs are used to interpolate the full CDF for each distribution from which quantiles can be extracted for fixed values (shown with open circles). The black line shows the resulting linear pool average ensemble. Inset shows corresponding PDFs.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;div xmlns=\u0026quot;http:\/\/www.w3.org\/1999\/xhtml\u0026quot;\u0026gt;(Panel A) Example of quantile output type predictions. In this example, points show model output collected for seven fixed quantile levels (\u0026lt;em\u0026gt;\u0026#x3B8;\u0026lt;\/em\u0026gt; = 0.01, 0.1, 0.3, 0.5, 0.7, 0.9, and 0.99) from two distributions (\u0026lt;em\u0026gt;N\u0026lt;\/em\u0026gt;(100, 10) in purple and \u0026lt;em\u0026gt;N\u0026lt;\/em\u0026gt;(120, 5) in green), with the underlying cumulative distribution functions (CDFs) shown with curves. The y-axis ticks show each of the fixed quantile levels. The associated values for each fixed quantile level do not align across distributions (vertical lines). (Panel B) Quantile average ensemble, which is calculated by averaging values for each fixed quantile level (represented by horizontal dashed gray lines). The distributions and corresponding model outputs from panel A are re-plotted and the black line shows the resulting quantile average ensemble. Inset shows corresponding probability density functions (PDFs). (Panel C) Linear pool ensemble, which is calculated by averaging cumulative probabilities for each fixed value (represented by vertical dashed gray lines). The distributions and corresponding model outputs from panel A are replotted. To calculate the linear pool in this case, where model outputs are not defined for the same values, the model outputs are used to interpolate the full CDF for each distribution from which quantiles can be extracted for fixed values (shown with open circles). The black line shows the resulting linear pool average ensemble. Inset shows corresponding PDFs.\u0026lt;\/div\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 1:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F1.medium.gif\u0022 width=\u0022440\u0022 height=\u0022175\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 1:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F1.medium.gif\u0022 width=\u0022440\u0022 height=\u0022175\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F1.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 1:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F1.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022 xmlns:xhtml=\u0022http:\/\/www.w3.org\/1999\/xhtml\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 1:\u003C\/span\u003E \u003Cp id=\u0022p-20\u0022 class=\u0022first-child\u0022\u003E(Panel A) Example of quantile output type predictions. In this example, points show model output collected for seven fixed quantile levels (\u003Cem\u003E\u03b8\u003C\/em\u003E = 0.01, 0.1, 0.3, 0.5, 0.7, 0.9, and 0.99) from two distributions (\u003Cem\u003EN\u003C\/em\u003E(100, 10) in purple and \u003Cem\u003EN\u003C\/em\u003E(120, 5) in green), with the underlying cumulative distribution functions (CDFs) shown with curves. The y-axis ticks show each of the fixed quantile levels. The associated values for each fixed quantile level do not align across distributions (vertical lines). (Panel B) Quantile average ensemble, which is calculated by averaging values for each fixed quantile level (represented by horizontal dashed gray lines). The distributions and corresponding model outputs from panel A are re-plotted and the black line shows the resulting quantile average ensemble. Inset shows corresponding probability density functions (PDFs). (Panel C) Linear pool ensemble, which is calculated by averaging cumulative probabilities for each fixed value (represented by vertical dashed gray lines). The distributions and corresponding model outputs from panel A are replotted. To calculate the linear pool in this case, where model outputs are not defined for the same values, the model outputs are used to interpolate the full CDF for each distribution from which quantiles can be extracted for fixed values (shown with open circles). The black line shows the resulting linear pool average ensemble. Inset shows corresponding PDFs.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-21\u0022\u003EThe probability average or linear pool is calculated by averaging probabilities across predictions for a fixed value of the target variable, \u003Cem\u003Ex\u003C\/em\u003E. In other words, for a set \u2131 = {\u003Cem\u003EF\u003C\/em\u003E\u003Csub\u003E\u003Cem\u003Ei\u003C\/em\u003E\u003C\/sub\u003E(\u003Cem\u003Ex\u003C\/em\u003E)|\u003Cem\u003Ei\u003C\/em\u003E \u2208 1, \u2026, \u003Cem\u003EN\u003C\/em\u003E} containing the values of CDFs at the point \u003Cem\u003Ex\u003C\/em\u003E and weights \u003Cstrong\u003E\u003Cem\u003Ew\u003C\/em\u003E\u003C\/strong\u003E, the linear pool is calculated as\n\u003Cspan class=\u0022disp-formula\u0022 id=\u0022disp-formula-3\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/graphic-4.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/graphic-4.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\n\n\u003C\/span\u003E\nFor a set of PMF values, {\u003Cem\u003Ef\u003C\/em\u003E\u003Csub\u003E\u003Cem\u003Ei\u003C\/em\u003E\u003C\/sub\u003E(\u003Cem\u003Ex\u003C\/em\u003E)|\u003Cem\u003Ei\u003C\/em\u003E \u2208 1, \u2026, \u003Cem\u003EN\u003C\/em\u003E}, the linear pool can be equivalently calculated: \u003Cspan class=\u0022inline-formula\u0022 id=\u0022inline-formula-3\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-3.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-3.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/span\u003E. Statistically this amounts to a mixture of the probability distributions, and the resulting probability distribution can be interpreted as one where the constituent probability distributions represent alternative predictions of the future, each of which has a probability \u003Cem\u003Ew\u003C\/em\u003E\u003Csub\u003E\u003Cem\u003Ei\u003C\/em\u003E\u003C\/sub\u003E of being the true one. For a visual depiction of these equations, see \u003Ca id=\u0022xref-fig-1-2\u0022 class=\u0022xref-fig\u0022 href=\u0022#F1\u0022\u003EFigure 1\u003C\/a\u003E below.\u003C\/p\u003E\u003Cp id=\u0022p-22\u0022\u003EThe different averaging methods for probabilistic predictions yield different properties of the resulting ensemble distribution. For example, the variance of the linear pool is \u003Cspan class=\u0022inline-formula\u0022 id=\u0022inline-formula-4\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-4.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-4.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/span\u003E \u003Cspan class=\u0022inline-formula\u0022 id=\u0022inline-formula-5\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-5.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-5.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/span\u003E where \u003Cem\u003E\u03bc\u003Csub\u003Ei\u003C\/sub\u003E\u003C\/em\u003E is the mean and \u003Cspan class=\u0022inline-formula\u0022 id=\u0022inline-formula-6\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-6.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-6.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/span\u003E is the variance of individual prediction \u003Cem\u003Ei\u003C\/em\u003E, and although there is no closed-form variance for the quantile average, the variance of the quantile average will always be less than or equal to that of the linear pool (\u003Ca id=\u0022xref-ref-15-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-15\u0022\u003ELichtendahl, Grushka-Cockayne, and Winkler 2013\u003C\/a\u003E). Both methods generate distributions with the same mean, \u003Cspan class=\u0022inline-formula\u0022 id=\u0022inline-formula-7\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-7.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/inline-graphic-7.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/span\u003E, which is the mean of individual model means (\u003Ca id=\u0022xref-ref-15-3\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-15\u0022\u003ELichtendahl, Grushka-Cockayne, and Winkler 2013\u003C\/a\u003E). The linear pool method preserves variation between individual models, whereas the quantile average cancels away this variation under the assumption it constitutes sampling error (\u003Ca id=\u0022xref-ref-13-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-13\u0022\u003EHowerton et al. 2023\u003C\/a\u003E).\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-3\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E3 Model implementation details\u003C\/h2\u003E\u003Cp id=\u0022p-23\u0022\u003ETo understand how these methods are implemented in \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E, we first must define the conventions employed by the hubverse and its packages for representing and working with model predictions. We begin with a short overview of concepts and conventions needed to utilize the \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package, supplemented by example predictions provided by the hubverse, then explain the implementation of the two ensembling functions included in the package, \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E and \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E.\u003C\/p\u003E\u003Cdiv id=\u0022sec-4\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E3.1 Hubverse terminology and conventions\u003C\/h3\u003E\u003Cp id=\u0022p-24\u0022\u003EA central concept in the hubverse effort is \u201cmodel output\u201d. Model output is a specially formatted tabular representation of predictions. Each row represents a single, unique prediction with each column providing information about what is being predicted, its scope, and its value. Per hubverse convention, each column serves one of three purposes: (i) denote which model has produced the prediction (called the \u201cmodel ID\u201d), (ii) provide details about what is being predicted (called the \u201ctask IDs\u201d), or (iii) specify the prediction itself and how it is represented (called the \u201cmodel output representation\u201d) (\u003Ca id=\u0022xref-ref-9-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-9\u0022\u003EConsortium of Infectious Disease Modeling Hubs 2024\u003C\/a\u003E).\u003C\/p\u003E\u003Cp id=\u0022p-25\u0022\u003EPredictions are assumed to be generated by distinct models, typically developed and run by a modeling team of one or more individuals. Each model should have a unique identifier that is stored in the \u003Ckbd\u003Emodel_id\u003C\/kbd\u003E column. Then, the details of the outcome being predicted can be stored in a series of task ID columns, the second type of column. These task ID columns may also include additional information, such as any conditions or assumptions that were used to generate the predictions (\u003Ca id=\u0022xref-ref-9-3\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-9\u0022\u003EConsortium of Infectious Disease Modeling Hubs 2024\u003C\/a\u003E). For example, short-term forecasts of incident influenza hospitalizations in the US at different locations and amounts of time in the future might represent this information using a \u003Ckbd\u003Etarget\u003C\/kbd\u003E column with the value \u201cwk inc flu hosp\u201d, a \u003Ckbd\u003Elocation\u003C\/kbd\u003E column identifying the location being predicted, a \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E column with the \u201cstarting point\u201d of the forecasts (not shown), and a \u003Ckbd\u003Ehorizon\u003C\/kbd\u003E column with the number of steps ahead that the forecast is predicting relative to the \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E (\u003Ca id=\u0022xref-table-wrap-1-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T1\u0022\u003ETable 1\u003C\/a\u003E). All these variables make up the task ID columns.\u003C\/p\u003E\u003Cdiv id=\u0022T1\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828941\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828941\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828941\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 1:\u003C\/span\u003E \u003Cp id=\u0022p-26\u0022 class=\u0022first-child\u0022\u003EExample of forecasts for incident influenza hospitalizations, formatted according to hubverse standards. Quantile predictions for the median and 50% prediction intervals from a single model are shown for four distinct horizons. The \u003Ckbd\u003Elocation\u003C\/kbd\u003E and \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E columns have been omitted for brevity; all forecasts in this example were made on 2022-12-17 for Massachusetts. This example is a modified subset of the \u003Ckbd\u003Eforecast_outputs\u003C\/kbd\u003E data provided by the \u003Ckbd\u003EhubExamples\u003C\/kbd\u003E package.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-27\u0022\u003EAlternatively, longer-term scenario projections may require different task ID columns. For example, projections of incident COVID-19 cases in the US at different locations, amounts of time in the future, and under different assumed conditions may use a \u003Ckbd\u003Etarget\u003C\/kbd\u003E column of \u201cinc case\u201d, a \u003Ckbd\u003Elocation\u003C\/kbd\u003E column specifying the location being predicted, an \u003Ckbd\u003Eorigin_date\u003C\/kbd\u003E column specifying the date on which the projections were made (not shown), a \u003Ckbd\u003Ehorizon\u003C\/kbd\u003E column describing the number of steps ahead that the projection is predicting relative to the \u003Ckbd\u003Eorigin_date\u003C\/kbd\u003E, and a \u003Ckbd\u003Escenario_id\u003C\/kbd\u003E column denoting the future conditions that were modeled and are projected to result in the specified number of incident cases (\u003Ca id=\u0022xref-table-wrap-2-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T2\u0022\u003ETable 2\u003C\/a\u003E). Different modeling efforts may use different sets of task ID columns and values to specify their prediction goals, or may simply choose distinct names to represent the same concept (e.g., \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E versus \u003Ckbd\u003Eorigin_date\u003C\/kbd\u003E for a date task ID). Additional examples of task ID variables are available on the hubverse documentation website (\u003Ca id=\u0022xref-ref-9-4\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-9\u0022\u003EConsortium of Infectious Disease Modeling Hubs 2024\u003C\/a\u003E).\u003C\/p\u003E\u003Cdiv id=\u0022T2\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828918\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828918\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828918\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 2:\u003C\/span\u003E \u003Cp id=\u0022p-28\u0022 class=\u0022first-child\u0022\u003EExample of scenario projections for incident COVID-19 cases, formatted according to hubverse standards. Quantile predictions for the median and 50% prediction intervals from a single model are shown for four distinct scenarios. The \u003Ckbd\u003Elocation\u003C\/kbd\u003E and \u003Ckbd\u003Eorigin_date\u003C\/kbd\u003E columns have been omitted for brevity; all forecasts in this example were made on 2021-03-07 for the US. This example is a modified subset of the \u003Ckbd\u003Escenario_outputs\u003C\/kbd\u003E data provided by the \u003Ckbd\u003EhubExamples\u003C\/kbd\u003E package.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-29\u0022\u003EThe third group of columns in model output specify the model predictions and details about how the predictions are represented. This \u201cmodel output representation\u201d includes the pre-dicted values along with metadata that specifies how the predictions are conveyed and always consists of the same three columns: (1) \u003Ckbd\u003Eoutput_type\u003C\/kbd\u003E, (2) \u003Ckbd\u003Eoutput_type_id\u003C\/kbd\u003E, and (3) \u003Ckbd\u003Evalue\u003C\/kbd\u003E. The \u003Ckbd\u003Eoutput_type\u003C\/kbd\u003E column defines how the prediction is represented and may be one of \u003Ckbd\u003E\u0022mean\u0022\u003C\/kbd\u003E or \u003Ckbd\u003E\u0022median\u0022\u003C\/kbd\u003E for point predictions, or \u003Ckbd\u003E\u0022quantile\u0022\u003C\/kbd\u003E, \u003Ckbd\u003E\u0022cdf\u0022\u003C\/kbd\u003E, \u003Ckbd\u003E\u0022pmf\u0022\u003C\/kbd\u003E, or \u003Ckbd\u003E\u0022sample\u0022\u003C\/kbd\u003E for probabilistic predictions (although the sample output type is not yet directly supported by the \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package and should be converted to another output type before computing an ensemble). The \u003Ckbd\u003Eoutput_type_id\u003C\/kbd\u003E provides additional identifying information for a prediction and is specific to the particular \u003Ckbd\u003Eoutput_type\u003C\/kbd\u003E (see \u003Ca id=\u0022xref-table-wrap-3-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T3\u0022\u003ETable 3\u003C\/a\u003E). For quantile predictions, the \u003Ckbd\u003Eoutput_type_id\u003C\/kbd\u003E is a numeric value between 0 and 1 specifying the cumulative probability associated with the quantile prediction. In the notation we defined above, the \u003Ckbd\u003Eoutput_type_id\u003C\/kbd\u003E corresponds to \u003Cem\u003E\u03b8\u003C\/em\u003E and the \u003Ckbd\u003Evalue\u003C\/kbd\u003E is the quantile prediction \u003Cem\u003EF\u003C\/em\u003E \u003Csup\u003E\u22121\u003C\/sup\u003E(\u003Cem\u003E\u03b8\u003C\/em\u003E). For CDF or PMF predictions, the \u003Ckbd\u003Eoutput_type_id\u003C\/kbd\u003E is the target variable value \u003Cem\u003Ex\u003C\/em\u003E at which the cumulative distribution function or probability mass function for the predictive distribution should be evaluated, and the \u003Ckbd\u003Evalue\u003C\/kbd\u003E column contains the predicted \u003Cem\u003EF\u003C\/em\u003E (\u003Cem\u003Ex\u003C\/em\u003E) or \u003Cem\u003Ef\u003C\/em\u003E(\u003Cem\u003Ex\u003C\/em\u003E), respectively. Requirements for the values of the \u003Ckbd\u003Eoutput_type_id\u003C\/kbd\u003E and \u003Ckbd\u003Evalue\u003C\/kbd\u003E columns associated with each valid output type are summarized in \u003Ca id=\u0022xref-table-wrap-3-2\u0022 class=\u0022xref-table\u0022 href=\u0022#T3\u0022\u003ETable 3\u003C\/a\u003E.\u003C\/p\u003E\u003Cdiv id=\u0022T3\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828940\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828940\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828940\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 3:\u003C\/span\u003E \u003Cp id=\u0022p-30\u0022 class=\u0022first-child\u0022\u003EA table summarizing how the model output representation columns are used for predictions of different output types. *Sample output types are not yet supported by the \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package. Adapted from \u003Ca href=\u0022https:\/\/hubverse.io\/en\/latest\/user-guide\/model-output.html#formats-of-model-output\u0022\u003Ehttps:\/\/hubverse.io\/en\/latest\/user-guide\/model-output.html#formats-of-model-output\u003C\/a\u003E\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-31\u0022\u003EThis representation of predictive model output is codified by the \u003Ckbd\u003Emodel_out_tbl\u003C\/kbd\u003E S3 class in the \u003Ckbd\u003EhubUtils,\u003C\/kbd\u003E package, one of the foundational hubverse packages. Although this S3 class is required for all \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E functions, model predictions in other formats can easily be transformed using the \u003Ckbd\u003Eas_model_out_tbl()\u003C\/kbd\u003E function from \u003Ckbd\u003EhubUtils\u003C\/kbd\u003E. An example of this transformation is provided in \u003Ca id=\u0022xref-sec-14-2\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-14\u0022\u003ESection 5\u003C\/a\u003E.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-5\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E3.2 Ensemble functions in hubEnsembles\u003C\/h3\u003E\u003Cp id=\u0022p-32\u0022\u003EThe \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package includes two functions that perform ensemble calculations: \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E, which applies some function to each model prediction, and \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E, which computes an ensemble using the linear opinion pool method. In the following sections, we outline the implementation details for each function and how these implementations correspond to the statistical ensembling methods described in \u003Ca id=\u0022xref-sec-2-2\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-2\u0022\u003ESection 2\u003C\/a\u003E. A short description of the calculation performed by each function is summarized by output type in \u003Ca id=\u0022xref-table-wrap-4-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T4\u0022\u003ETable 4\u003C\/a\u003E.\u003C\/p\u003E\u003Cdiv id=\u0022T4\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828923\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828923\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828923\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 4:\u003C\/span\u003E \u003Cp id=\u0022p-33\u0022 class=\u0022first-child\u0022\u003ESummary of ensemble function calculations for each output type. The ensemble function determines the operation that is performed, and in the case of probabilistic output types (quantile, CDF, PMF), this also determines what ensemble distribution is generated (quantile average, \u003Cem\u003EF\u003C\/em\u003E \u003Csup\u003E\u22121\u003C\/sup\u003E(\u003Cem\u003E\u03b8\u003C\/em\u003E), or linear pool, \u003Cem\u003EF\u003Csub\u003ELOP\u003C\/sub\u003E\u003C\/em\u003E (\u003Cem\u003Ex\u003C\/em\u003E)). The resulting ensemble will be returned in the same output type as the inputs. Thus, the output type determines how the resulting ensemble distribution is summarized (as a quantile, \u003Cem\u003EF\u003C\/em\u003E \u003Csup\u003E\u22121\u003C\/sup\u003E(\u003Cem\u003E\u03b8\u003C\/em\u003E), cumulative probability, \u003Cem\u003EF\u003C\/em\u003E (\u003Cem\u003Ex\u003C\/em\u003E), or probability \u003Cem\u003Ef\u003C\/em\u003E(\u003Cem\u003Ex\u003C\/em\u003E)). Estimating individual model cumulative probabilities is required to compute a \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E for predictions of \u003Ckbd\u003Equantile\u003C\/kbd\u003E output type; see \u003Ca id=\u0022xref-sec-7-1\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-7\u0022\u003ESection 3.2.2\u003C\/a\u003E on the linear pool operation for details. In the case of \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E, we report the calculations for the default case where \u003Ckbd\u003Eagg_fun =mean\u003C\/kbd\u003E ; however, if another aggregation function is chosen (e.g., \u003Ckbd\u003Eagg_fun=median\u003C\/kbd\u003E), that calculation would be performed instead. For example, \u003Ckbd\u003Esimple_ensemble(\u2026, agg_fun=median)\u003C\/kbd\u003E applied to predictions of mean output type would return the median of individual model means.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-6\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E3.2.1 Simple ensemble\u003C\/h4\u003E\u003Cp id=\u0022p-34\u0022\u003EThe \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E function directly computes an ensemble from component model out-puts by combining them via some function (\u003Cem\u003EC\u003C\/em\u003E) within each unique combination of task ID variables, output types, and output type IDs. This function can be used to summarize predictions of output types mean, median, quantile, CDF, and PMF. The mechanics of the ensemble calculations are the same for each of the output types, though the resulting statistical ensembling method differs for different output types (\u003Ca id=\u0022xref-table-wrap-4-2\u0022 class=\u0022xref-table\u0022 href=\u0022#T4\u0022\u003ETable 4\u003C\/a\u003E).\u003C\/p\u003E\u003Cp id=\u0022p-35\u0022\u003EBy default, \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E uses the mean for the aggregation function \u003Cem\u003EC\u003C\/em\u003E and equal weights for all models. For point predictions with a mean or median output type, the resulting ensemble prediction is an equally weighted average of the individual models\u2019 predictions. For probabilistic predictions in a quantile format, by default \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E calculates an equally weighted average of individual model target variable values at each quantile level, which is equivalent to a quantile average. For model outputs in a CDF or PMF format, by default \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E computes an equally weighted average of individual model (cumulative or bin) probabilities at each target variable value, which is equivalent to the linear pool method.\u003C\/p\u003E\u003Cp id=\u0022p-36\u0022\u003EAny aggregation function \u003Cem\u003EC\u003C\/em\u003E may be specified by the user. For example, a median ensemble may also be created by specifying \u003Ckbd\u003Emedian\u003C\/kbd\u003E as the aggregation function, or a custom function may be passed to the \u003Ckbd\u003Eagg_fun\u003C\/kbd\u003E argument to create other ensemble types. Similarly, model weights can be specified to create a weighted ensemble.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-7\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E3.2.2 Linear pool\u003C\/h4\u003E\u003Cp id=\u0022p-37\u0022\u003EThe \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E function implements the linear opinion pool (LOP) method for ensembling predictions. Currently, this function can be used to combine predictions with output types mean, quantile, CDF, and PMF. Unlike \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E, this function handles its computation differently based on the output type. For the CDF, PMF, and mean output types, the linear pool method is equivalent to calling \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E with a mean aggregation function (see \u003Ca id=\u0022xref-table-wrap-4-3\u0022 class=\u0022xref-table\u0022 href=\u0022#T4\u0022\u003ETable 4\u003C\/a\u003E), since \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E produces a linear pool prediction (an average of individual model cumulative or bin probabilities).\u003C\/p\u003E\u003Cp id=\u0022p-38\u0022\u003EHowever, implementation of LOP is less straightforward for the quantile output type. This is because LOP averages cumulative probabilities at each value of the target variable, but the predictions are quantiles (on the scale of the target variable) for fixed quantile levels. The value for these quantile predictions will generally differ between models, and as a result we are typically not provided cumulative probabilities at the same values of the target variable for all component predictions. This lack of alignment between cumulative probabilities for the same target variable values impedes computation of LOP from quantile predictions and is illustrated in panel A of \u003Ca id=\u0022xref-fig-1-3\u0022 class=\u0022xref-fig\u0022 href=\u0022#F1\u0022\u003EFigure 1\u003C\/a\u003E.\u003C\/p\u003E\u003Cp\u003EGiven that LOP cannot be directly calculated from quantile predictions, we must first obtain an estimate of the CDF for each component distribution from the provided quantiles, combine the CDFs, then calculate the quantiles using the ensemble\u2019s CDF. We perform this calculation in three main steps, assisted by the \u003Ckbd\u003Edistfromq\u003C\/kbd\u003E package (\u003Ca id=\u0022xref-ref-21-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-21\u0022\u003ERay and Gerding 2024\u003C\/a\u003E) for the first two:\n\u003C\/p\u003E\u003Col class=\u0022list-ord \u0022 id=\u0022list-1\u0022\u003E\u003Cli id=\u0022list-item-1\u0022\u003E\u003Cp id=\u0022p-40\u0022\u003EInterpolate and extrapolate from the provided quantiles for each component model to obtain an estimate of the CDF of that particular distribution.\u003C\/p\u003E\u003C\/li\u003E\u003Cli id=\u0022list-item-2\u0022\u003E\u003Cp id=\u0022p-41\u0022\u003EDraw samples from each component model distribution. To reduce Monte Carlo variability, we use quasi-random samples corresponding to quantiles of the estimated distribution (\u003Ca id=\u0022xref-ref-17-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-17\u0022\u003ENiederreiter 1992\u003C\/a\u003E).\u003C\/p\u003E\u003C\/li\u003E\u003Cli id=\u0022list-item-3\u0022\u003E\u003Cp id=\u0022p-42\u0022\u003EPool the samples from all component models and extract the desired quantiles.\u003C\/p\u003E\u003C\/li\u003E\u003C\/ol\u003E\u003Cp\u003E\nFor step 1, functionality in the \u003Ckbd\u003Edistfromq\u003C\/kbd\u003E package uses a monotonic cubic spline for interpolation on the interior of the provided quantiles. The user may choose one of several distributions to perform extrapolation of the CDF tails. These include normal, lognormal, and cauchy distributions, with \u201cnormal\u201d set as the default. A location-scale parameterization is used, with separate location and scale parameters chosen in the lower and upper tails so as to match the two most extreme quantiles. The sampling process described in steps 2 and 3 approximates the linear pool calculation described in \u003Ca id=\u0022xref-sec-2-3\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-2\u0022\u003ESection 2\u003C\/a\u003E.\u003C\/p\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-8\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E4 Basic demonstration of functionality\u003C\/h2\u003E\u003Cp id=\u0022p-43\u0022\u003EIn this section, we provide a simple example to illustrate the two main functions in \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E, \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E and \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E.\u003C\/p\u003E\u003Cdiv id=\u0022sec-9\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E4.1 Example data: a forecast hub\u003C\/h3\u003E\u003Cp id=\u0022p-44\u0022\u003EWe will use an example hub provided by the hubverse to demonstrate the functionality of the \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package (\u003Ca id=\u0022xref-ref-9-5\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-9\u0022\u003EConsortium of Infectious Disease Modeling Hubs 2024\u003C\/a\u003E). This example hub was generated with modified forecasts from the FluSight forecasting challenge, which will be discussed in detail in \u003Ca id=\u0022xref-sec-14-3\u0022 class=\u0022xref-sec\u0022 href=\u0022#sec-14\u0022\u003ESection 5\u003C\/a\u003E. The example hub includes both example model output data and target data (sometimes known as \u201ctruth\u201d data), which are stored in the \u003Ckbd\u003EhubExamples\u003C\/kbd\u003E package as data objects named \u003Ckbd\u003Eforecast_outputs\u003C\/kbd\u003E and \u003Ckbd\u003Eforecast_target_ts\u003C\/kbd\u003E. Note that the toy model outputs contain predictions for only a small subset rows of select dates, locations, and output type IDs, far fewer than an actual modeling hub would typically collect.\u003C\/p\u003E\u003Cp id=\u0022p-45\u0022\u003EThe model output data includes quantile, mean and median forecasts of future incident influenza hospitalizations and PMF forecasts of hospitalization intensity. Each forecast is made for five task ID variables, including the location for which the forecast was made (\u003Ckbd\u003Elocation\u003C\/kbd\u003E), the date on which the forecast was made (\u003Ckbd\u003Ereference_date\u003C\/kbd\u003E), the number of steps ahead (\u003Ckbd\u003Ehorizon\u003C\/kbd\u003E), the date of the forecast prediction (a combination of the date the forecast was made and the forecast horizon, \u003Ckbd\u003Etarget_end_date\u003C\/kbd\u003E), and the forecast target (\u003Ckbd\u003Etarget\u003C\/kbd\u003E). \u003Ca id=\u0022xref-table-wrap-5-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T5\u0022\u003ETable 5\u003C\/a\u003E provides an example set of quantile forecasts included in this example model output. In \u003Ca id=\u0022xref-table-wrap-5-2\u0022 class=\u0022xref-table\u0022 href=\u0022#T5\u0022\u003ETable 5\u003C\/a\u003E, we show only the median, the 50%, and 90% prediction intervals, although other intervals and mean forecasts are included in the example model output data.\u003C\/p\u003E\u003Cdiv id=\u0022T5\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828937\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828937\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828937\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 5:\u003C\/span\u003E \u003Cp id=\u0022p-46\u0022 class=\u0022first-child\u0022\u003EExample model output for forecasts of incident influenza hospitalizations. A subset of example model output is shown: 1-week ahead quantile forecasts made on 2022-12-17 for Massachusetts from three distinct models; only the median and 5th, 25th, 75th and 95th quantiles are displayed. The \u003Ckbd\u003Elocation\u003C\/kbd\u003E, \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E and \u003Ckbd\u003Etarget_end_date\u003C\/kbd\u003E columns have been omitted for brevity. This example data is provided in the \u003Ckbd\u003EhubExamples\u003C\/kbd\u003E package.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-47\u0022\u003EWe also have corresponding target data included in the \u003Ckbd\u003EhubExamples\u003C\/kbd\u003E package (\u003Ca id=\u0022xref-table-wrap-6-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T6\u0022\u003ETable 6\u003C\/a\u003E). The example target data provide observed incident influenza hospitalizations (\u003Ckbd\u003Eobservation\u003C\/kbd\u003E) in a given week (\u003Ckbd\u003Edate\u003C\/kbd\u003E) and for a given location (\u003Ckbd\u003Elocation\u003C\/kbd\u003E). This target data could be used as calibration data for generating forecasts or for evaluating the forecasts post hoc. The forecast-specific task ID variables \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E and \u003Ckbd\u003Ehorizon\u003C\/kbd\u003E are not relevant for the target data.\u003C\/p\u003E\u003Cdiv id=\u0022T6\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828933\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828933\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828933\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 6:\u003C\/span\u003E \u003Cp id=\u0022p-48\u0022 class=\u0022first-child\u0022\u003EExample target data for incident influenza hospitalizations. This table includes target data from 2022-11-01 and 2023-02-01. This target data is provided in the \u003Ckbd\u003EhubExamples\u003C\/kbd\u003E package.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-49\u0022\u003EWe can plot these forecasts and the target data using the \u003Ckbd\u003Eplot_step_ahead_model_output()\u003C\/kbd\u003E function from \u003Ckbd\u003EhubVis\u003C\/kbd\u003E, another package for visualizing model outputs from the hubverse suite (\u003Ca id=\u0022xref-fig-2-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F2\u0022\u003EFigure 2\u003C\/a\u003E). We subset the model output data and the target data to the location and time horizons we are interested in.\u003C\/p\u003E\u003Cdiv id=\u0022F2\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F2.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022One example set of quantile forecasts of weekly incident influenza hospitalizations in Massachusetts from each of three models (panels). Forecasts are represented by a median (line), 50% and 90% prediction intervals (ribbons). Gray points represent observed incident hospitalizations.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;One example set of quantile forecasts of weekly incident influenza hospitalizations in Massachusetts from each of three models (panels). Forecasts are represented by a median (line), 50% and 90% prediction intervals (ribbons). Gray points represent observed incident hospitalizations.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 2:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F2.medium.gif\u0022 width=\u0022440\u0022 height=\u0022213\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 2:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F2.medium.gif\u0022 width=\u0022440\u0022 height=\u0022213\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F2.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 2:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F2.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 2:\u003C\/span\u003E \u003Cp id=\u0022p-50\u0022 class=\u0022first-child\u0022\u003EOne example set of quantile forecasts of weekly incident influenza hospitalizations in Massachusetts from each of three models (panels). Forecasts are represented by a median (line), 50% and 90% prediction intervals (ribbons). Gray points represent observed incident hospitalizations.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022F3\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-12.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-12.medium.gif\u0022 width=\u0022440\u0022 height=\u0022227\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-12.medium.gif\u0022 width=\u0022440\u0022 height=\u0022227\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-12.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download \u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-12.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-13.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-13.medium.gif\u0022 width=\u0022440\u0022 height=\u0022122\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-13.medium.gif\u0022 width=\u0022440\u0022 height=\u0022122\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-13.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download \u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F3\/graphic-13.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-51\u0022\u003ENext, we examine the PMF target in the example model output data. For this target, teams forecasted the probability that hospitalization intensity will be \u201clow\u201d, \u201cmoderate\u201d, \u201chigh\u201d, or \u201cvery high\u201d. These hospitalization intensity categories are determined by thresholds for weekly hospital admissions per 100,000 population. In other words, \u201clow\u201d hospitalization intensity in a given week means few incident influenza hospitalizations per 100,000 population are predicted, whereas \u201cvery high\u201d hospitalization intensity means many hospitalizations per 100,000 population are predicted. These forecasts are made for the same task ID variables as the \u003Ckbd\u003Equantile\u003C\/kbd\u003E forecasts of incident hospitalizations, other than the target, which is \u201cwk flu hosp rate category\u201d for these categorical predictions.\u003C\/p\u003E\u003Cp id=\u0022p-52\u0022\u003EWe show a representative example of the hospitalization intensity category forecasts in \u003Ca id=\u0022xref-table-wrap-7-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T7\u0022\u003ETable 7\u003C\/a\u003E. Because these forecasts are PMF output type, the \u003Ckbd\u003Eoutput_type_id\u003C\/kbd\u003E column specifies the bin of hospitalization intensity and the \u003Ckbd\u003Evalue\u003C\/kbd\u003E column provides the forecasted probability of hospitalization incidence being in that category. Values sum to 1 across bins. For the MOBS-GLEAM_FLUH and PSI-DICE models, incidence is forecasted to decrease over the horizon (\u003Ca id=\u0022xref-fig-2-2\u0022 class=\u0022xref-fig\u0022 href=\u0022#F2\u0022\u003EFigure 2\u003C\/a\u003E), and correspondingly, there is lower probability of \u201chigh\u201d and \u201cvery high\u201d hospitalization intensity for later horizons (\u003Ca id=\u0022xref-fig-4-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F4\u0022\u003EFigure 3\u003C\/a\u003E).\u003C\/p\u003E\u003Cdiv id=\u0022T7\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828932\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828932\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828932\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 7:\u003C\/span\u003E \u003Cp id=\u0022p-53\u0022 class=\u0022first-child\u0022\u003EExample PMF model output for forecasts of incident influenza hospitalization intensity. A subset of example model output is shown: 1-week ahead PMF forecasts made on 2022-12-17 for Massachusetts from three distinct models. We round the forecasted probability (in the \u003Ckbd\u003Evalue\u003C\/kbd\u003E column) to two digits. The \u003Ckbd\u003Elocation\u003C\/kbd\u003E, \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E and \u003Ckbd\u003Etarget_end_date\u003C\/kbd\u003E columns have been omitted for brevity. This example data is provided in the \u003Ckbd\u003EhubExamples\u003C\/kbd\u003E package.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022F4\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F4.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022One example PMF forecast of incident influenza hospitalization intensity is shown for each of three models (panels). Each cell shows the forecasted probability of a given hospitalization intensity bin (low, moderate, high, and very high) for each forecast horizon (0-3 weeks ahead). Darker colors indicate higher forecasted probability.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;One example PMF forecast of incident influenza hospitalization intensity is shown for each of three models (panels). Each cell shows the forecasted probability of a given hospitalization intensity bin (low, moderate, high, and very high) for each forecast horizon (0-3 weeks ahead). Darker colors indicate higher forecasted probability.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 3:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F4.medium.gif\u0022 width=\u0022440\u0022 height=\u0022211\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 3:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F4.medium.gif\u0022 width=\u0022440\u0022 height=\u0022211\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F4.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 3:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F4.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 3:\u003C\/span\u003E \u003Cp id=\u0022p-54\u0022 class=\u0022first-child\u0022\u003EOne example PMF forecast of incident influenza hospitalization intensity is shown for each of three models (panels). Each cell shows the forecasted probability of a given hospitalization intensity bin (low, moderate, high, and very high) for each forecast horizon (0-3 weeks ahead). Darker colors indicate higher forecasted probability.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-10\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E4.2 Creating ensembles with simple_ensemble\u003C\/h3\u003E\u003Cp id=\u0022p-55\u0022\u003EUsing the default options for \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E, we can generate an equally weighted mean ensemble for each unique combination of values for the task ID variables, the \u003Ckbd\u003Eoutput_type\u003C\/kbd\u003E and the \u003Ckbd\u003Eoutput_type_id\u003C\/kbd\u003E. Recall that this means different ensemble methods will be used for different output types: for the \u003Ckbd\u003Equantile\u003C\/kbd\u003E output type in our example data, the resulting ensemble is a quantile average, while for the PMF output type, the ensemble is a linear pool.\u003C\/p\u003E\u003Cdiv id=\u0022F5\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F5.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F5.medium.gif\u0022 width=\u0022440\u0022 height=\u002268\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F5.medium.gif\u0022 width=\u0022440\u0022 height=\u002268\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F5.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure5\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F5.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-56\u0022\u003EThe resulting model output has the same structure as the original model output data (\u003Ca id=\u0022xref-table-wrap-8-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T8\u0022\u003ETable 8\u003C\/a\u003E), with columns for model ID, task ID variables, output type, output type ID, and value. We also use \u003Ckbd\u003Emodel_id=\u0022simple-ensemble-mean\u0022\u003C\/kbd\u003E to change the name of this ensemble in the resulting model output; if not specified, the default will be \u201chub-ensemble\u201d.\u003C\/p\u003E\u003Cdiv id=\u0022T8\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828914\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828914\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828914\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 8:\u003C\/span\u003E \u003Cp id=\u0022p-57\u0022 class=\u0022first-child\u0022\u003EMean ensemble model output. The values in the \u003Ckbd\u003Emodel_id\u003C\/kbd\u003E column are set by the argument \u003Ckbd\u003Esimple_ensemble(\u2026,model_id)\u003C\/kbd\u003E. Results are generated for all output types, but only a subset are shown: 1-week ahead forecasts made on 2022-12-17 for Massachusetts, with only the median, 25th and 75th quantiles for the quantile output type and all bins for the PMF output type. The \u003Ckbd\u003Elocation\u003C\/kbd\u003E, \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E and \u003Ckbd\u003Etarget_end_date\u003C\/kbd\u003E columns have been omitted for brevity, and the \u003Ckbd\u003Evalue\u003C\/kbd\u003E column is rounded to two digits.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-11\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E4.2.1 Changing the aggregation function\u003C\/h4\u003E\u003Cp id=\u0022p-58\u0022\u003EWe can change the function that is used to aggregate model outputs. For example, we may want to calculate a median of the component models\u2019 submitted values for each quantile. We do so by specifying \u003Ckbd\u003Eagg_fun=median\u003C\/kbd\u003E.\u003C\/p\u003E\u003Cdiv id=\u0022F6\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F6.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F6.medium.gif\u0022 width=\u0022440\u0022 height=\u002281\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F6.medium.gif\u0022 width=\u0022440\u0022 height=\u002281\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F6.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure6\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F6.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-59\u0022\u003ECustom functions can also be passed into the \u003Ckbd\u003Eagg_fun\u003C\/kbd\u003E argument. We illustrate this by defining a custom function to compute the ensemble prediction as a geometric mean of the component model predictions. Any custom function to be used must have an argument \u003Ckbd\u003Ex\u003C\/kbd\u003E for the vector of numeric values to summarize, and if relevant, an argument \u003Ckbd\u003Ew\u003C\/kbd\u003E of numeric weights.\u003C\/p\u003E\u003Cdiv id=\u0022F7\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F7.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F7.medium.gif\u0022 width=\u0022440\u0022 height=\u0022134\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F7.medium.gif\u0022 width=\u0022440\u0022 height=\u0022134\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F7.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure7\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F7.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-60\u0022\u003EAs expected, the mean, median, and geometric mean each give us slightly different resulting ensembles. The median point estimates, 50% prediction intervals, and 90% prediction intervals in \u003Ca id=\u0022xref-fig-8-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F8\u0022\u003EFigure 4\u003C\/a\u003E demonstrate this.\u003C\/p\u003E\u003Cdiv id=\u0022F8\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F8.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022Three different ensembles for weekly incident influenza hospitalizations in Mas-sachusetts. Each ensemble combines individual predictions from the example hub (Figure 2) using a different method: arithmetic mean, geometric mean, or median. All methods correspond to variations of the quantile average approach. Ensembles are represented by a median (line), 50% and 90% prediction intervals (ribbons). Geometric mean ensemble and simple mean ensemble generate similar estimates in this case.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;Three different ensembles for weekly incident influenza hospitalizations in Mas-sachusetts. Each ensemble combines individual predictions from the example hub (Figure 2) using a different method: arithmetic mean, geometric mean, or median. All methods correspond to variations of the quantile average approach. Ensembles are represented by a median (line), 50% and 90% prediction intervals (ribbons). Geometric mean ensemble and simple mean ensemble generate similar estimates in this case.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 4:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F8.medium.gif\u0022 width=\u0022440\u0022 height=\u0022217\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 4:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F8.medium.gif\u0022 width=\u0022440\u0022 height=\u0022217\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F8.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 4:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F8.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 4:\u003C\/span\u003E \u003Cp id=\u0022p-61\u0022 class=\u0022first-child\u0022\u003EThree different ensembles for weekly incident influenza hospitalizations in Mas-sachusetts. Each ensemble combines individual predictions from the example hub (\u003Ca id=\u0022xref-fig-2-3\u0022 class=\u0022xref-fig\u0022 href=\u0022#F2\u0022\u003EFigure 2\u003C\/a\u003E) using a different method: arithmetic mean, geometric mean, or median. All methods correspond to variations of the quantile average approach. Ensembles are represented by a median (line), 50% and 90% prediction intervals (ribbons). Geometric mean ensemble and simple mean ensemble generate similar estimates in this case.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-12\u0022 class=\u0022subsection\u0022\u003E\u003Ch4\u003E4.2.2 Weighting model contributions\u003C\/h4\u003E\u003Cp id=\u0022p-62\u0022\u003EWe can weight the contributions of each model in the ensemble using the \u003Ckbd\u003Eweights\u003C\/kbd\u003E argument of \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E. This argument takes a \u003Ckbd\u003Edata.frame\u003C\/kbd\u003E that should include a \u003Ckbd\u003Emodel_id\u003C\/kbd\u003E column containing each unique model ID and a \u003Ckbd\u003Eweight\u003C\/kbd\u003E column. In the following example, we include the baseline model in the ensemble, but give it less weight than the other forecasts.\u003C\/p\u003E\u003Cdiv id=\u0022F9\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F9.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F9.medium.gif\u0022 width=\u0022440\u0022 height=\u0022134\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F9.medium.gif\u0022 width=\u0022440\u0022 height=\u0022134\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F9.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure9\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F9.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-13\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E4.3 Creating ensembles with linear_pool\u003C\/h3\u003E\u003Cp id=\u0022p-63\u0022\u003EWe can also generate a linear pool ensemble, or distributional mixture, using the \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E function; this function can be applied to predictions with an \u003Ckbd\u003Eoutput_type\u003C\/kbd\u003E of mean, quantile, CDF, or PMF. Our example hub includes median output type, so we exclude it from the calculation.\u003C\/p\u003E\u003Cdiv id=\u0022F10\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F10.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F10.medium.gif\u0022 width=\u0022440\u0022 height=\u002243\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F10.medium.gif\u0022 width=\u0022440\u0022 height=\u002243\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F10.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure10\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F10.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-64\u0022\u003EAs described above, for \u003Ckbd\u003Equantile\u003C\/kbd\u003E model outputs, the \u003Ckbd\u003Elinear_pool\u003C\/kbd\u003E function approximates the full probability distribution for each component prediction using the value-quantile pairs provided by that model, and then obtains quasi-random samples from that distributional estimate. The number of samples drawn from the distribution of each component model defaults to \u003Ckbd\u003E1e4\u003C\/kbd\u003E, but this can be changed using the \u003Ckbd\u003En_samples\u003C\/kbd\u003E argument.\u003C\/p\u003E\u003Cp id=\u0022p-65\u0022\u003EIn \u003Ca id=\u0022xref-fig-11-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F11\u0022\u003EFigure 5\u003C\/a\u003E, we compare ensemble results generated by \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E and \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E for model outputs of output types PMF and quantile. As expected, the results from the two functions are equivalent for the PMF output type: for this output type, the \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E method averages the predicted probability of each category across the component models, which is the definition of the linear pool ensemble method. This is not the case for the quantile output type, because the \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E is computing a quantile average.\u003C\/p\u003E\u003Cdiv id=\u0022F11\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F11.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022Comparison of results from linear_pool() (blue) and simple_ensemble() (red). (Panel A) Ensemble predictions of Massachusetts incident influenza hospitalization intensity (classified as low, moderate, high, or very high), which provide an example of PMF output type. (Panel B) Ensemble predictions of weekly incident influenza hospitalizations in Massachusetts, which provide an example of quantile output type. Note, for quantile output type, simple_ensemble() corresponds to a quantile average. Ensembles combine individual models from the example hub, and are represented by a median (line), 50% and 90% prediction intervals (ribbons) (Figure 2).\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;div xmlns=\u0026quot;http:\/\/www.w3.org\/1999\/xhtml\u0026quot;\u0026gt;Comparison of results from \u0026lt;kbd\u0026gt;linear_pool()\u0026lt;\/kbd\u0026gt; (blue) and \u0026lt;kbd\u0026gt;simple_ensemble()\u0026lt;\/kbd\u0026gt; (red). (Panel A) Ensemble predictions of Massachusetts incident influenza hospitalization intensity (classified as low, moderate, high, or very high), which provide an example of PMF output type. (Panel B) Ensemble predictions of weekly incident influenza hospitalizations in Massachusetts, which provide an example of quantile output type. Note, for quantile output type, \u0026lt;kbd\u0026gt;simple_ensemble()\u0026lt;\/kbd\u0026gt; corresponds to a quantile average. Ensembles combine individual models from the example hub, and are represented by a median (line), 50% and 90% prediction intervals (ribbons) (Figure 2).\u0026lt;\/div\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 5:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F11.medium.gif\u0022 width=\u0022440\u0022 height=\u0022175\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 5:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F11.medium.gif\u0022 width=\u0022440\u0022 height=\u0022175\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F11.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 5:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F11.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 5:\u003C\/span\u003E \u003Cp id=\u0022p-66\u0022 class=\u0022first-child\u0022\u003EComparison of results from \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E (blue) and \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E (red). (Panel A) Ensemble predictions of Massachusetts incident influenza hospitalization intensity (classified as low, moderate, high, or very high), which provide an example of PMF output type. (Panel B) Ensemble predictions of weekly incident influenza hospitalizations in Massachusetts, which provide an example of quantile output type. Note, for quantile output type, \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E corresponds to a quantile average. Ensembles combine individual models from the example hub, and are represented by a median (line), 50% and 90% prediction intervals (ribbons) (\u003Ca id=\u0022xref-fig-2-4\u0022 class=\u0022xref-fig\u0022 href=\u0022#F2\u0022\u003EFigure 2\u003C\/a\u003E).\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-14\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E5 Example: in-depth analysis of forecast data\u003C\/h2\u003E\u003Cp id=\u0022p-67\u0022\u003ETo further demonstrate the utility of the \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package and the differences between the two ensembling functions, we examine a more complex example. Unlike the previous section\u2019s basic showcase of functionality, we use this case study to provide a more complete analysis that compares and evaluates ensemble model performance using real forecasts collected by a modeling hub, with an overarching goal of choosing a single best ensembling approach for the application.\u003C\/p\u003E\u003Cp id=\u0022p-68\u0022\u003ESince 2013, the US Centers for Disease Control and Prevention (CDC) has been soliciting forecasts of seasonal influenza from modeling teams through a collaborative challenge called FluSight (\u003Ca id=\u0022xref-ref-6-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-6\u0022\u003ECDC 2023\u003C\/a\u003E). We use a subset of these predictions to create four equally-weighted ensembles with \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E and \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E and compare the resulting ensembles\u2019 performance. The ensembling methods chosen for this case study consist of a quantile (arithmetic) mean, a quantile median, a linear pool with normal tails, and a linear pool with log-normal tails. Note that only a select portion of the code is shown in this manuscript for brevity, although all the functions and scripts used to generate the case study results can be found in the associated GitHub repository (\u003Ca href=\u0022https:\/\/github.com\/hubverse-org\/hubEnsemblesManuscript\u0022\u003Ehttps:\/\/github.com\/hubverse-org\/hubEnsemblesManuscript\u003C\/a\u003E). More specifically, the figures and tables supporting this analysis are generated reproducibly using data from rds files stored in the \u003Ckbd\u003Eanalysis\/data\/raw-data\u003C\/kbd\u003E directory and scripts in the \u003Ckbd\u003Einst\u003C\/kbd\u003E directory of the repository.\u003C\/p\u003E\u003Cdiv id=\u0022sec-15\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E5.1 Data and Methods\u003C\/h3\u003E\u003Cp id=\u0022p-69\u0022\u003EWe begin by querying the component forecasts used to generate the four ensembles from Zoltar (\u003Ca id=\u0022xref-ref-23-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-23\u0022\u003EReich et al. 2021\u003C\/a\u003E), a repository designed to archive forecasts created by the Reich Lab at UMass Amherst. For this analysis we only consider FluSight predictions in a quantile format from the 2021-2022 and 2022-2023 seasons. These forecasts were stored in two data objects, split by season, called \u003Ckbd\u003Eflu_forecasts-zoltar_21-22.rds\u003C\/kbd\u003E and \u003Ckbd\u003Eflu_forecasts-zoltar_22-23.rds\u003C\/kbd\u003E, and a subset is shown below in \u003Ca id=\u0022xref-table-wrap-9-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T9\u0022\u003ETable 9\u003C\/a\u003E.\u003C\/p\u003E\u003Cdiv id=\u0022T9\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828929\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828929\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828929\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 9:\u003C\/span\u003E \u003Cp id=\u0022p-70\u0022 class=\u0022first-child\u0022\u003EMean ensemble model output. The values in the \u003Ckbd\u003Emodel_id\u003C\/kbd\u003E column are set by the argument \u003Ckbd\u003Esimple_ensemble(\u2026,model_id)\u003C\/kbd\u003E. Results are generated for all output types, but only a subset are shown: 1-week ahead forecasts made on 2022-12-17 for Massachusetts, with only the median, 25th and 75th quantiles for the quantile output type and all bins for the PMF output type. The \u003Ckbd\u003Elocation\u003C\/kbd\u003E, \u003Ckbd\u003Ereference_date\u003C\/kbd\u003E and \u003Ckbd\u003Etarget_end_date\u003C\/kbd\u003E columns have been omitted for brevity, and the \u003Ckbd\u003Evalue\u003C\/kbd\u003E column is rounded to two digits.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022F12\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F12.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F12.medium.gif\u0022 width=\u0022440\u0022 height=\u002295\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F12.medium.gif\u0022 width=\u0022440\u0022 height=\u002295\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F12.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure12\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F12.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-71\u0022\u003EForecasts must conform to hubverse standards to be fed into either of the ensembling functions, so we first transform the raw forecasts using the \u003Ckbd\u003Eas_model_out_tbl()\u003C\/kbd\u003E \u003Csup\u003E\u003Ca id=\u0022xref-fn-1-1\u0022 class=\u0022xref-fn\u0022 href=\u0022#fn-1\u0022\u003E1\u003C\/a\u003E\u003C\/sup\u003E function from the \u003Ckbd\u003EhubUtils\u003C\/kbd\u003E package. Here, we specify the task ID variables \u003Ckbd\u003Eforecast_date\u003C\/kbd\u003E (when the forecast was made), \u003Ckbd\u003Elocation\u003C\/kbd\u003E, \u003Ckbd\u003Ehorizon\u003C\/kbd\u003E, and \u003Ckbd\u003Etarget\u003C\/kbd\u003E.\u003C\/p\u003E\u003Cdiv id=\u0022F13\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-26.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-26.medium.gif\u0022 width=\u0022440\u0022 height=\u0022175\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-26.medium.gif\u0022 width=\u0022440\u0022 height=\u0022175\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-26.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download \u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-26.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-27.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-27.medium.gif\u0022 width=\u0022440\u0022 height=\u002281\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-27.medium.gif\u0022 width=\u0022440\u0022 height=\u002281\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-27.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download \u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F13\/graphic-27.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-72\u0022\u003EPrior to ensemble calculation (shown later in this section), we filter out any predictions (defined by a unique combination of task ID variables) that did not include all 23 quantiles specified by FluSight (\u003Cem\u003E\u03b8\u003C\/em\u003E \u2208 {.010, 0.025, .050, .100, \u2026, .900, .950, .990}). The FluSight baseline and median ensemble models generated by the FluSight hub are also excluded from the component forecasts. We chose to remove the baseline to match the composition of models used to create the official FluSight ensemble.\u003C\/p\u003E\u003Cp id=\u0022p-73\u0022\u003EWith these inclusion criteria, the final data set of component forecasts consists of predictions from 25 modeling teams and 42 distinct models, 53 forecast dates (one per week), 54 US locations, 4 horizons, 1 target, and 23 quantiles. In the 2021-2022 season, 25 models made predictions for 22 weeks spanning from late January 2022 to late June 2022, and in the 2022-2023 season, there were 31 models making predictions for 31 weeks spanning mid-October 2022 to mid-May 2023. Fourteen of the 42 total models made forecasts for both seasons.\u003C\/p\u003E\u003Cp id=\u0022p-74\u0022\u003EIn both seasons, forecasts were made for the same locations (the 50 US states, Washington DC, Puerto Rico, the Virgin Islands, and the US as a whole), horizons (1 to 4 weeks ahead), quantiles (the 23 described above), and target (week ahead incident flu hospitalization). The values for the forecasts are always non-negative. In \u003Ca id=\u0022xref-table-wrap-10-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T10\u0022\u003ETable 10\u003C\/a\u003E, we provide an example of these predictions, showing select quantiles from a single model, forecast date, horizon, and location.\u003C\/p\u003E\u003Cdiv id=\u0022T10\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828919\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828919\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828919\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 10:\u003C\/span\u003E \u003Cp id=\u0022p-75\u0022 class=\u0022first-child\u0022\u003EAn example prediction of weekly incident influenza hospitalizations. The example model output was made on May 15, 2023 for California at the 1 week ahead horizon. The forecast was generated during the FluSight forecasting challenge, then formatted according to hubverse standards post hoc. The \u003Ckbd\u003Elocation\u003C\/kbd\u003E, \u003Ckbd\u003Eforecast_date\u003C\/kbd\u003E, and \u003Ckbd\u003Eseason\u003C\/kbd\u003E columns have been omitted for brevity; quantiles representing the endpoints of the central 50%, 80% and 95% prediction intervals are shown.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-76\u0022\u003ENext, we combine the component model outputs to generate predictions from each ensemble model. We begin by excluding the baseline model from the set of predictions that will be combined. Then, we create one object to store the ensemble results generated from each method we are interested in comparing.\u003C\/p\u003E\u003Cdiv id=\u0022F14\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F14.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F14.medium.gif\u0022 width=\u0022440\u0022 height=\u0022411\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F14.medium.gif\u0022 width=\u0022440\u0022 height=\u0022411\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F14.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure14\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F14.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-77\u0022\u003EWe evaluate the performance of these ensembles using scoring metrics that measure the accuracy and calibration of their forecasts. Here, we choose several common metrics in forecast evaluation, including mean absolute error (MAE), weighted interval score (WIS) (\u003Ca id=\u0022xref-ref-5-1\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-5\u0022\u003EBracher et al. 2021\u003C\/a\u003E), 50% prediction interval (PI) coverage, and 95% PI coverage. MAE measures the average absolute error of a set of point forecasts; smaller values of MAE indicate better forecast accuracy. WIS is a generalization of MAE for probabilistic forecasts and is an alternative to other common proper scoring rules which cannot be evaluated directly for quantile forecasts (\u003Ca id=\u0022xref-ref-5-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-5\u0022\u003EBracher et al. 2021\u003C\/a\u003E). WIS is made up of three component penalties: (1) for over-prediction, (2) for under-prediction, and (3) for the spread of each interval (where an interval is defined by a symmetric set of two quantiles). This metric weights these penalties across all prediction intervals provided. A lower WIS value indicates a more accurate forecast (\u003Ca id=\u0022xref-ref-5-3\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-5\u0022\u003EBracher et al. 2021\u003C\/a\u003E). PI coverage provides information about whether a forecast has accurately characterized its uncertainty about future observations. The 50% PI coverage rate measures the proportion of the time that 50% prediction intervals at that nominal level included the observed value; the 95% PI coverage rate is defined similarly. Achieving approximately nominal (50% or 95%) coverage indicates a well-calibrated forecast.\u003C\/p\u003E\u003Cp id=\u0022p-78\u0022\u003EWe also use relative versions of WIS and MAE (rWIS and rMAE, respectively) to understand how the ensemble performance compares to that of the FluSight baseline model. These metrics are calculated as\n\u003Cspan class=\u0022disp-formula\u0022 id=\u0022disp-formula-4\u0022\u003E\u003Cspan class=\u0022highwire-responsive-lazyload\u0022\u003E\u003Cimg src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 class=\u0022highwire-embed lazyload\u0022 alt=\u0022Embedded Image\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/graphic-30.gif\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-embed\u0022 alt=\u0022Embedded Image\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/highwire\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/embed\/graphic-30.gif\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\n\n\u003C\/span\u003E\nwhere model \u003Cem\u003Em\u003C\/em\u003E is any given model being compared against the baseline. For both of these metrics, a value less than one indicates better performance compared to the baseline while a value greater than one indicates worse performance. By definition, the FluSight baseline itself will always have a value of one for both of these metrics.\u003C\/p\u003E\u003Cp id=\u0022p-79\u0022\u003EEach unique prediction from an ensemble model is scored against target data using the \u003Ckbd\u003Escore_forecasts()\u003C\/kbd\u003E \u003Csup\u003E\u003Ca id=\u0022xref-fn-2-1\u0022 class=\u0022xref-fn\u0022 href=\u0022#fn-2\u0022\u003E2\u003C\/a\u003E\u003C\/sup\u003E function from the \u003Ckbd\u003EcovidHubUtils\u003C\/kbd\u003E package, as a hubverse package for scoring and evaluation has not yet been fully implemented. This function outputs each of the metrics described above. We use median forecasts taken from the 0.5 quantile for the MAE evaluation.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv id=\u0022sec-16\u0022 class=\u0022subsection\u0022\u003E\u003Ch3\u003E5.2 Performance results across ensembles\u003C\/h3\u003E\u003Cp id=\u0022p-80\u0022\u003EThe quantile median ensemble has the best overall performance in terms of WIS and MAE (and the relative versions of these metrics), and has coverage rates that were close to the nominal levels (\u003Ca id=\u0022xref-table-wrap-11-1\u0022 class=\u0022xref-table\u0022 href=\u0022#T11\u0022\u003ETable 11\u003C\/a\u003E). The two linear opinion pools have very similar performance to each other. These methods have the second-best performance as measured by WIS and MAE, but they have the highest 50% and 95% coverage rates, with empirical coverage that was well above the nominal coverage rate. The quantile mean performs the worst of the ensembles with the highest MAE, which is substantially different from that of the other ensembles.\u003C\/p\u003E\u003Cdiv id=\u0022T11\u0022 class=\u0022table pos-float\u0022\u003E\u003Cdiv class=\u0022table-inline table-callout-links\u0022\u003E\u003Cdiv class=\u0022callout\u0022\u003E\u003Cspan\u003EView this table:\u003C\/span\u003E\u003Cul class=\u0022callout-links\u0022\u003E\u003Cli class=\u0022view-inline first\u0022\u003E\u003Ca href=\u0022\u0022 class=\u0022table-expand-inline\u0022 data-table-url=\u0022\/highwire\/markup\/828922\/expansion?postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0026amp;table-expand-inline=1\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView inline\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022view-popup\u0022\u003E\u003Ca href=\u0022\/highwire\/markup\/828922\/expansion?width=1000\u0026amp;height=500\u0026amp;iframe=true\u0026amp;postprocessors=highwire_tables%2Chighwire_reclass%2Chighwire_figures%2Chighwire_math%2Chighwire_inline_linked_media%2Chighwire_embed\u0022 class=\u0022colorbox colorbox-load table-expand-popup\u0022 rel=\u0022gallery-fragment-tables\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EView popup\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022download-ppt last\u0022\u003E\u003Ca href=\u0022\/highwire\/powerpoint\/828922\u0022 class=\u0022highwire-figure-link highwire-figure-link-ppt\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload powerpoint\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022table-caption\u0022\u003E\u003Cspan class=\u0022table-label\u0022\u003ETable 11:\u003C\/span\u003E \u003Cp id=\u0022p-81\u0022 class=\u0022first-child\u0022\u003ESummary of overall model performance across both seasons, averaged over all locations except the US national location. The best value for each metric is bolded, though the metric values are often quite similar among the models.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-82\u0022\u003EPlots of the models\u2019 forecasts can aid our understanding about the origin of these accuracy differences. For example, the linear opinion pools consistently have some of the widest prediction intervals, and consequently the highest coverage rates. The median ensemble, which has the best WIS, balanced interval width with calibration best overall, with narrower intervals than the linear pools that still achieved near-nominal coverage on average across all time points. The quantile mean\u2019s interval widths vary, though it usually has narrower intervals than the linear pools. However, this model\u2019s point forecasts have a larger error margin compared to the other ensembles, especially at longer horizons. This pattern is demonstrated in \u003Ca id=\u0022xref-fig-15-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F15\u0022\u003EFigure 6\u003C\/a\u003E for the 4-week ahead forecast in California following the 2022-23 season peak on December 5, 2022. Here the quantile mean predicted a continued increase in hospitalizations, at a steeper slope than the other ensemble methods.\u003C\/p\u003E\u003Cdiv id=\u0022F15\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F15.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022One to four week ahead forecasts for select dates plotted against target data for California. The first panel shows all models on the same scale. All other panels show forecasts for each individual model,2w7ith varying y-axis scales, and their prediction accuracy as compared to observed influenza hospitalizations.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;One to four week ahead forecasts for select dates plotted against target data for California. The first panel shows all models on the same scale. All other panels show forecasts for each individual model,2w7ith varying y-axis scales, and their prediction accuracy as compared to observed influenza hospitalizations.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 6:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F15.medium.gif\u0022 width=\u0022325\u0022 height=\u0022440\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 6:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F15.medium.gif\u0022 width=\u0022325\u0022 height=\u0022440\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F15.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 6:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F15.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 6:\u003C\/span\u003E \u003Cp id=\u0022p-83\u0022 class=\u0022first-child\u0022\u003EOne to four week ahead forecasts for select dates plotted against target data for California. The first panel shows all models on the same scale. All other panels show forecasts for each individual model,2w7ith varying y-axis scales, and their prediction accuracy as compared to observed influenza hospitalizations.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-84\u0022\u003EAveraging across all time points, the median model can be seen to have the best scores for every metric. It outperforms the mean ensemble by a similar amount for both MAE and WIS, particularly around local times of change (see \u003Ca id=\u0022xref-fig-16-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F16\u0022\u003EFigure 7\u003C\/a\u003E and \u003Ca id=\u0022xref-fig-17-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F17\u0022\u003EFigure 8\u003C\/a\u003E). The median ensemble also has better coverage rates than the mean ensemble in the tails of the distribution (95% intervals, see \u003Ca id=\u0022xref-fig-18-1\u0022 class=\u0022xref-fig\u0022 href=\u0022#F18\u0022\u003EFigure 9\u003C\/a\u003E) and similar coverage in the center (50% intervals). The median model also outperforms the linear pools for most weeks, with the greatest differences in scores being for WIS and coverage rates (\u003Ca id=\u0022xref-fig-17-2\u0022 class=\u0022xref-fig\u0022 href=\u0022#F17\u0022\u003EFigure 8\u003C\/a\u003E and \u003Ca id=\u0022xref-fig-18-2\u0022 class=\u0022xref-fig\u0022 href=\u0022#F18\u0022\u003EFigure 9\u003C\/a\u003E). This seems to indicate that the linear pools\u2019 estimates are usually too conservative, with their wide intervals and higher-than-nominal coverage rates being penalized by WIS. However, during the 2022-2023 season there are several localized times when the linear pools showcased better one-week-ahead forecasts than the median ensemble (\u003Ca id=\u0022xref-fig-17-3\u0022 class=\u0022xref-fig\u0022 href=\u0022#F17\u0022\u003EFigure 8\u003C\/a\u003E). These localized instances are characterized by similar MAE values (\u003Ca id=\u0022xref-fig-17-4\u0022 class=\u0022xref-fig\u0022 href=\u0022#F17\u0022\u003EFigure 8\u003C\/a\u003E) for the two methods and poor median ensemble coverage rates (\u003Ca id=\u0022xref-fig-18-3\u0022 class=\u0022xref-fig\u0022 href=\u0022#F18\u0022\u003EFigure 9\u003C\/a\u003E). In these instances, the wide intervals from the linear pools were useful in capturing the eventually-observed hospitalizations, usually during times of rapid change.\u003C\/p\u003E\u003Cdiv id=\u0022F16\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F16.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022Mean absolute error (MAE) averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average MAE is shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Lower values indicate better performance.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;Mean absolute error (MAE) averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average MAE is shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Lower values indicate better performance.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 7:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F16.medium.gif\u0022 width=\u0022369\u0022 height=\u0022440\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 7:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F16.medium.gif\u0022 width=\u0022369\u0022 height=\u0022440\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F16.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 7:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F16.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 7:\u003C\/span\u003E \u003Cp id=\u0022p-85\u0022 class=\u0022first-child\u0022\u003EMean absolute error (MAE) averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average MAE is shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Lower values indicate better performance.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022F17\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F17.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u0022Weighted interval score (WIS) averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average WIS is shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Lower values indicate better performance.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;Weighted interval score (WIS) averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average WIS is shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Lower values indicate better performance.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 8:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F17.medium.gif\u0022 width=\u0022369\u0022 height=\u0022440\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 8:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F17.medium.gif\u0022 width=\u0022369\u0022 height=\u0022440\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F17.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 8:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F17.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 8:\u003C\/span\u003E \u003Cp id=\u0022p-86\u0022 class=\u0022first-child\u0022\u003EWeighted interval score (WIS) averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average WIS is shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Lower values indicate better performance.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv id=\u0022F18\u0022 class=\u0022fig pos-float type-figure  odd\u0022\u003E\u003Cdiv class=\u0022highwire-figure\u0022\u003E\u003Cdiv class=\u0022fig-inline-img-wrapper\u0022\u003E\u003Cdiv class=\u0022fig-inline-img\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F18.large.jpg?width=800\u0026amp;height=600\u0026amp;carousel=1\u0022 title=\u002295% prediction interval (PI) coverage averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average coverage is30shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Ideal coverage of 95% is shown (black horizontal line); values closer to 95% indicate better performance.\u0022 class=\u0022highwire-fragment fragment-images colorbox-load\u0022 rel=\u0022gallery-fragment-images-1435764067\u0022 data-figure-caption=\u0022\u0026lt;div class=\u0026quot;highwire-markup\u0026quot;\u0026gt;95% prediction interval (PI) coverage averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average coverage is30shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Ideal coverage of 95% is shown (black horizontal line); values closer to 95% indicate better performance.\u0026lt;\/div\u0026gt;\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003E\u003Cspan class=\u0022hw-responsive-img\u0022\u003E\u003Cimg class=\u0022highwire-fragment fragment-image lazyload\u0022 alt=\u0022Figure 9:\u0022 src=\u0022data:image\/gif;base64,R0lGODlhAQABAIAAAAAAAP\/\/\/yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\u0022 data-src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F18.medium.gif\u0022 width=\u0022331\u0022 height=\u0022440\u0022\/\u003E\u003Cnoscript\u003E\u003Cimg class=\u0022highwire-fragment fragment-image\u0022 alt=\u0022Figure 9:\u0022 src=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F18.medium.gif\u0022 width=\u0022331\u0022 height=\u0022440\u0022\/\u003E\u003C\/noscript\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cul class=\u0022highwire-figure-links inline\u0022\u003E\u003Cli class=\u0022download-fig first\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F18.large.jpg?download=true\u0022 class=\u0022highwire-figure-link highwire-figure-link-download\u0022 title=\u0022Download Figure 9:\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EDownload figure\u003C\/a\u003E\u003C\/li\u003E\u003Cli class=\u0022new-tab last\u0022\u003E\u003Ca href=\u0022https:\/\/www.medrxiv.org\/content\/medrxiv\/early\/2024\/06\/25\/2024.06.24.24309416\/F18.large.jpg\u0022 class=\u0022highwire-figure-link highwire-figure-link-newtab\u0022 target=\u0022_blank\u0022 data-icon-position=\u0022\u0022 data-hide-link-title=\u00220\u0022\u003EOpen in new tab\u003C\/a\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022fig-caption\u0022\u003E\u003Cspan class=\u0022fig-label\u0022\u003EFigure 9:\u003C\/span\u003E \u003Cp id=\u0022p-87\u0022 class=\u0022first-child\u0022\u003E95% prediction interval (PI) coverage averaged across all locations. Average target data across all locations for 2021-2022 (A) and 2022-2023 (B) seasons for reference. For each season, average coverage is30shown for 1-week (C-D) and 4-week ahead (E-F) forecasts. Results are plotted for each ensemble model (colors) across the entire season. Ideal coverage of 95% is shown (black horizontal line); values closer to 95% indicate better performance.\u003C\/p\u003E\u003Cdiv class=\u0022sb-div caption-clear\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cp id=\u0022p-88\u0022\u003EIn this analysis, all of the ensemble variations outperform the baseline model; yet, different ensembling methods perform best under different circumstances. While the quantile median has the best overall results for WIS, MAE, 50% PI coverage, and 95% PI coverage, other models may perform better from week-to-week for each metric. Around the 2022-2023 season\u2019s peak in early December, the remaining four models (including the baseline) each have instances in which they achieve the lowest WIS, like the linear pool ensembles for the one week ahead horizon over several weeks of this period.\u003C\/p\u003E\u003Cp id=\u0022p-89\u0022\u003EThe choice of an appropriate ensemble aggregation method may depend on the forecast target, the goal of forecasting, and the behavior of the individual models contributing to an ensemble. One case may call for prioritizing high coverage rates while another may prioritize accurate point forecasts. The \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E and \u003Ckbd\u003Elinear_pool()\u003C\/kbd\u003E functions and the ability to specify component model weights and an aggregation function for \u003Ckbd\u003Esimple_ensemble()\u003C\/kbd\u003E allow users to implement a variety of ensemble methods.\u003C\/p\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-17\u0022\u003E\u003Ch2 class=\u0022\u0022\u003E6 Summary and discussion\u003C\/h2\u003E\u003Cp id=\u0022p-90\u0022\u003EEnsembles of independent models are a powerful tool to generate more accurate and more reliable predictions of future outcomes than a single model alone. Here, we have demonstrated how to utilize \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E, a simple and flexible framework to combine individual model predictions into an ensemble.\u003C\/p\u003E\u003Cp id=\u0022p-91\u0022\u003EThe \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package is situated within the larger hubverse collection of open-source software and data tools to support collaborative modeling exercises (\u003Ca id=\u0022xref-ref-9-6\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-9\u0022\u003EConsortium of Infectious Disease Modeling Hubs 2024\u003C\/a\u003E). Collaborative hubs offer many benefits, including serving as a centralized entity to guide and elicit predictions from multiple independent models (\u003Ca id=\u0022xref-ref-24-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-24\u0022\u003EReich et al. 2022\u003C\/a\u003E). Given the increasing popularity of multi-model ensembles and collaborative hubs, there is a clear need for generalized data standards and software infrastructure to support these hubs. By addressing this need, the hubverse suite of tools can reduce duplicative efforts across existing hubs, support other communities engaged in collaborative efforts, and enable the adoption of multi-model approaches in new domains.\u003C\/p\u003E\u003Cp id=\u0022p-92\u0022\u003EWhen using \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E, it is important to carefully choose an ensemble method that is well suited for the situation. Although there may not be a universal \u201cbest\u201d method, matching the properties of a given ensemble method with the features of the component models will likely yield best results (\u003Ca id=\u0022xref-ref-13-2\u0022 class=\u0022xref-bibr\u0022 href=\u0022#ref-13\u0022\u003EHowerton et al. 2023\u003C\/a\u003E). Our case study on seasonal influenza forecasts in the US demonstrates this point. The quantile median ensemble performs best overall for a range of metrics, including weighted interval score, mean absolute error, and prediction interval coverage. Yet, the linear pool method, which generates an ensemble with wider prediction intervals, demonstrates performance advantages during periods of rapid change, when outlying component forecasts are likely more important. Notably, all ensemble methods outperform the baseline model. The performance improvements from ensemble models motivate the use of a \u201chub-based\u201d approach to prediction for infectious diseases and in other fields.\u003C\/p\u003E\u003Cp id=\u0022p-93\u0022\u003EOngoing development of the \u003Ckbd\u003EhubEnsembles\u003C\/kbd\u003E package and the larger suite of hubverse tools will continue to support multi-model predictions in new ways, including for example supporting additional types of predictions, enabling scoring and evaluation of those predictions, and allowing for cloud-based data storage. All such infrastructure will ultimately provide a comprehensive suite of open-source software tools for leveraging the power of collaborative hubs and multi-model ensembles.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section data-availability\u0022 id=\u0022sec-18\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EData Availability\u003C\/h2\u003E\u003Cp id=\u0022p-94\u0022\u003EAll code and data necessary to reproduce this manuscript can be found at https:\/\/github.com\/hubverse-org\/hubEnsemblesManuscript. The related R package can be found at https:\/\/github.com\/hubverse-org\/hubEnsembles.\u003C\/p\u003E\u003Cp id=\u0022p-95\u0022\u003E\n\u003Ca href=\u0022https:\/\/github.com\/hubverse-org\/hubEnsemblesManuscript\u0022\u003Ehttps:\/\/github.com\/hubverse-org\/hubEnsemblesManuscript\u003C\/a\u003E\n\u003C\/p\u003E\u003Cp id=\u0022p-96\u0022\u003E\n\u003Ca href=\u0022https:\/\/github.com\/hubverse-org\/hubEnsembles\u0022\u003Ehttps:\/\/github.com\/hubverse-org\/hubEnsembles\u003C\/a\u003E\n\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section\u0022 id=\u0022sec-19\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EConsortium of Infectious Disease Modeling Hubs\u003C\/h2\u003E\u003Cp id=\u0022p-98\u0022\u003EConsortium of Infectious Disease Modeling Hubs authors include Alvaro J. Castro Rivadeneira (University of Massachusetts Amherst), Lucie Contamin (University of Pittsburgh), Sebastian Funk (London School of Hygiene \u0026amp; Tropical Medicine), Aaron Gerding (University of Massachusetts Amherst), Hugo Gruson (\u003Ca href=\u0022http:\/\/data.org\u0022\u003Edata.org\u003C\/a\u003E), Harry Hochheiser (University of Pittsburgh), Emily Howerton (The Pennsylvania State University), Melissa Kerr (University of Massachusetts Amherst), Anna Krystalli (R-RSE SMPC), Sara L. Loo (Johns Hopkins University), Evan L. Ray (University of Massachusetts Amherst), Nicholas G. Reich (University of Massachusetts Amherst), Koji Sato (Johns Hopkins University), Li Shandross (University of Massachusetts Amherst), Katharine Sherratt (London School of Hygene and Tropical Medicine), Shaun Truelove (Johns Hopkins University), Martha Zorn (University of Massachusetts Amherst)\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section ack\u0022 id=\u0022ack-1\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EAcknowledgements\u003C\/h2\u003E\u003Cp id=\u0022p-97\u0022\u003EThe authors thank all members of the hubverse community; the broader hubverse software infrastructure made this package possible. L. Shandross, A. Krystalli, N. G. Reich, and E. L. Ray were supported by the National Institutes of General Medical Sciences (R35GM119582) and the US Centers for Disease Control and Prevention (U01IP001122 and NU38FT000008). E. Howerton was supported by the Eberly College of Science Barbara McClintock Science Achievement Graduate Scholarship in Biology at the Pennsylvania State University. L. Contamin and H. Hochheiser were supported by NIGMS grant U24GM132013. The content is solely the responsibility of the authors and does not necessarily represent the official views of NIGMS, the National Institutes of Health, or CDC.\u003C\/p\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section fn-group\u0022 id=\u0022fn-group-1\u0022\u003E\u003Ch2\u003EFootnotes\u003C\/h2\u003E\u003Cul\u003E\u003Cli class=\u0022fn\u0022 id=\u0022fn-1\u0022\u003E\u003Cp id=\u0022p-99\u0022\u003E\u003Ca class=\u0022rev-xref\u0022 href=\u0022#xref-fn-1-1\u0022\u003E\u21b5\u003C\/a\u003E\u003Cspan class=\u0022fn-label\u0022\u003E\u003Csup\u003E1\u003C\/sup\u003E\u003C\/span\u003E \u003Ca href=\u0022https:\/\/hubverse-org.github.io\/hubUtils\/reference\/as_model_out_tbl.html\u0022\u003Ehttps:\/\/hubverse-org.github.io\/hubUtils\/reference\/as_model_out_tbl.html\u003C\/a\u003E\u003C\/p\u003E\u003C\/li\u003E\u003Cli class=\u0022fn\u0022 id=\u0022fn-2\u0022\u003E\u003Cp id=\u0022p-100\u0022\u003E\u003Ca class=\u0022rev-xref\u0022 href=\u0022#xref-fn-2-1\u0022\u003E\u21b5\u003C\/a\u003E\u003Cspan class=\u0022fn-label\u0022\u003E\u003Csup\u003E2\u003C\/sup\u003E\u003C\/span\u003E \u003Ca href=\u0022https:\/\/reichlab.io\/covidHubUtils\/reference\/score_forecasts.html\u0022\u003Ehttps:\/\/reichlab.io\/covidHubUtils\/reference\/score_forecasts.html\u003C\/a\u003E\u003C\/p\u003E\u003C\/li\u003E\u003C\/ul\u003E\u003C\/div\u003E\u003Cdiv class=\u0022section ref-list\u0022 id=\u0022ref-list-1\u0022\u003E\u003Ch2 class=\u0022\u0022\u003EReferences\u003C\/h2\u003E\u003Col class=\u0022cit-list ref-use-labels\u0022\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-1-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-1\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-web\u0022 id=\u0022cit-2024.06.24.24309416v1.1\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EAastveit\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKnut Are\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJames\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EMitchell\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EFrancesco\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ERavazzolo\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EHerman K.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003Evan Dijk\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2018\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EThe Evolution of Forecast Density Combinations in Economics\u003C\/span\u003E.\u201d \u003Cspan class=\u0022cit-source\u0022\u003ETinbergen Institute Discussion Papers\u003C\/span\u003E. \u003Ca href=\u0022https:\/\/hdl.handle.net\/10419\/185588\u0022\u003Ehttps:\/\/hdl.handle.net\/10419\/185588\u003C\/a\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-2-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-2\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.2\u0022 data-doi=\u002210.1126\/science.aav7274\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EAlley\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERichard B.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKerry A.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EEmanuel\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EFuqing\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EZhang\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2019\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EAdvances in Weather Prediction\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EScience\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E363\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E6425\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E342\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E44\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1126\/science.aav7274\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DScience%26rft.stitle%253DScience%26rft.aulast%253DAlley%26rft.auinit1%253DR.%2BB.%26rft.volume%253D363%26rft.issue%253D6425%26rft.spage%253D342%26rft.epage%253D344%26rft.atitle%253DAdvances%2Bin%2Bweather%2Bprediction%26rft_id%253Dinfo%253Adoi%252F10.1126%252Fscience.aav7274%26rft_id%253Dinfo%253Apmid%252F30679358%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/ijlink\/YTozOntzOjQ6InBhdGgiO3M6MTQ6Ii9sb29rdXAvaWpsaW5rIjtzOjU6InF1ZXJ5IjthOjQ6e3M6ODoibGlua1R5cGUiO3M6NDoiQUJTVCI7czoxMToiam91cm5hbENvZGUiO3M6Mzoic2NpIjtzOjU6InJlc2lkIjtzOjEyOiIzNjMvNjQyNS8zNDIiO3M6NDoiYXRvbSI7czo1MDoiL21lZHJ4aXYvZWFybHkvMjAyNC8wNi8yNS8yMDI0LjA2LjI0LjI0MzA5NDE2LmF0b20iO31zOjg6ImZyYWdtZW50IjtzOjA6IiI7fQ==\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-ijlink\u0022\u003E\u003Cspan\u003E\u003Cspan class=\u0022cit-reflinks-abstract\u0022\u003EAbstract\u003C\/span\u003E\u003Cspan class=\u0022cit-sep cit-reflinks-variant-name-sep\u0022\u003E\/\u003C\/span\u003E\u003Cspan class=\u0022cit-reflinks-full-text\u0022\u003E\u003Cspan class=\u0022free-full-text\u0022\u003EFREE \u003C\/span\u003EFull Text\u003C\/span\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-3-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-3\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.3\u0022 data-doi=\u002210.1016\/j.epidem.2023.100705\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EBorchering\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERebecca K.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJessica M.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EHealy\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EBetsy L.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ECadwell\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMichael A.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EJohansson\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERachel B.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ESlayton\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMegan\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EWallace\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMatthew\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBiggerstaff\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EPublic Health Impact of the U.S. Scenario Modeling Hub\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EEpidemics\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E44\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003ESeptember\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E100705\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1016\/j.epidem.2023.100705\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DEpidemics%26rft.volume%253D44%26rft.spage%253D100705%26rft_id%253Dinfo%253Adoi%252F10.1016%252Fj.epidem.2023.100705%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1016\/j.epidem.2023.100705\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-4-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-4\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.4\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EBosse\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ENikos\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EYuling\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EYao\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ESam\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EAbbott\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ESebastian\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EFunk\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003EStackr: Create Mixture Models From Predictive Samples\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-5-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-5\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.5\u0022 data-doi=\u002210.1371\/journal.pcbi.1008618\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EBracher\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJohannes\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEvan L.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ERay\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ETilmann\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EGneiting\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ENicholas G.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EReich\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2021\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EEvaluating Epidemic Forecasts in an Interval Format\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EPLOS Computational Biology\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E17\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E2\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003Ee1008618\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1371\/journal.pcbi.1008618\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DPLOS%2BComputational%2BBiology%26rft.volume%253D17%26rft.spage%253De1008618%26rft_id%253Dinfo%253Adoi%252F10.1371%252Fjournal.pcbi.1008618%26rft_id%253Dinfo%253Apmid%252F33577550%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1371\/journal.pcbi.1008618\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=33577550\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F25%2F2024.06.24.24309416.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-6-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-6\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-web\u0022 id=\u0022cit-2024.06.24.24309416v1.6\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth cit-collab\u0022\u003ECDC\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EAbout Flu Forecasting\u003C\/span\u003E.\u201d \u003Ca href=\u0022https:\/\/www.cdc.gov\/flu\/weekly\/flusight\/how-flu-forecasting.htm\u0022\u003Ehttps:\/\/www.cdc.gov\/flu\/weekly\/flusight\/how-flu-forecasting.htm\u003C\/a\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-7-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-7\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.7\u0022 data-doi=\u002210.1016\/0169-2070(89)90012-5\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EClemen\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERobert T\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E1989\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003ECombining Forecasts: A Review and Annotated Bibliography\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EInternational Journal of Forecasting\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E5\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E4\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E559\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E83\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1016\/0169-2070(89)90012-5\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DInternational%2BJournal%2Bof%2BForecasting%26rft.volume%253D5%26rft.spage%253D559%26rft_id%253Dinfo%253Adoi%252F10.1016%252F0169-2070%252889%252990012-5%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1016\/0169-2070(89)90012-5\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-8-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-8\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.8\u0022 data-doi=\u002210.1371\/journal.pmed.1003542\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ECol\u00f3n-Gonz\u00e1lez\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EFelipe J.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ELeonardo Soares\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBastos\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EBarbara\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EHofmann\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAlison\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EHopkin\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EQuillon\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EHarpham\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ETom\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ECrocker\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERosanna\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EAmato\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2021\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EProbabilistic Seasonal Dengue Forecasting in Vietnam: A Modelling Study Using Superensembles\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EPLOS Medicine\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E18\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E3\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003Ee1003542\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1371\/journal.pmed.1003542\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DPLOS%2BMedicine%26rft.volume%253D18%26rft.spage%253D1003542e%26rft_id%253Dinfo%253Adoi%252F10.1371%252Fjournal.pmed.1003542%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1371\/journal.pmed.1003542\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-9-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-9\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-web\u0022 id=\u0022cit-2024.06.24.24309416v1.9\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth cit-collab\u0022\u003EConsortium of Infectious Disease Modeling Hubs\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2024\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EThe Hubverse: Open Tools for Collaborative Forecasting\u003C\/span\u003E.\u201d \u003Ca href=\u0022https:\/\/hubdocs.readthedocs.io\/en\/latest\/index.html\u0022\u003Ehttps:\/\/hubdocs.readthedocs.io\/en\/latest\/index.html\u003C\/a\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-10-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-10\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.10\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ECouch\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ESimon\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMax\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EKuhn\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EStacks: Tidy Model Stacking\u003C\/abbr\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-11-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-11\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.11\u0022 data-doi=\u002210.1073\/pnas.2113561119\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ECramer\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEstee Y.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEvan L.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ERay\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EVelma K.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ELopez\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJohannes\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBracher\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAndrea\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBrennen\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAlvaro J.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ECastro Rivadeneira\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAaron\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EGerding\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2022\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EEvaluation of Individual and Ensemble Probabilistic Forecasts of COVID-19 Mortality in the United States\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EProceedings of the National Academy of Sciences\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E119\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E15\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003Ee2113561119\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1073\/pnas.2113561119\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DProceedings%2Bof%2Bthe%2BNational%2BAcademy%2Bof%2BSciences%26rft.volume%253D119%26rft.spage%253De2113561119%26rft_id%253Dinfo%253Adoi%252F10.1073%252Fpnas.2113561119%26rft_id%253Dinfo%253Apmid%252F35394862%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1073\/pnas.2113561119\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=35394862\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F25%2F2024.06.24.24309416.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-12-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-12\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.12\u0022 data-doi=\u002210.1016\/j.ijforecast.2004.05.002\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EHibon\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMich\u00e8le\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ETheodoros\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EEvgeniou\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2005\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003ETo Combine or Not to Combine: Selecting Among Forecasts and Their Combinations\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EInternational Journal of Forecasting\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E21\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E1\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E15\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E24\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1016\/j.ijforecast.2004.05.002\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DInternational%2BJournal%2Bof%2BForecasting%26rft.volume%253D21%26rft.spage%253D15%26rft_id%253Dinfo%253Adoi%252F10.1016%252Fj.ijforecast.2004.05.002%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1016\/j.ijforecast.2004.05.002\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-13-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-13\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.13\u0022 data-doi=\u002210.1098\/rsif.2022.0659\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EHowerton\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEmily\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMichael C.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ERunge\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ETiffany L.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBogich\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERebecca K.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBorchering\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EHidetoshi\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EIn-amine\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJustin\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ELessler\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ELuke C.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EMullany\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EContext-Dependent Representation of Within- and Between-Model Uncertainty: Aggregating Probabilistic Predictions in Infectious Disease Epidemiology\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJournal of The Royal Society Interface\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E20\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E198\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E20220659\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1098\/rsif.2022.0659\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJournal%2Bof%2BThe%2BRoyal%2BSociety%2BInterface%26rft.volume%253D20%26rft.spage%253D20220659%26rft_id%253Dinfo%253Adoi%252F10.1098%252Frsif.2022.0659%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1098\/rsif.2022.0659\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-14-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-14\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.14\u0022 data-doi=\u002210.1073\/pnas.1909865116\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EJohansson\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMichael A.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKaryn M.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EApfeldorf\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EScott\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EDobson\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJason\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EDevita\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAnna L.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBuczak\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EBenjamin\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBaugher\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ELinda J.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EMoniz\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2019\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EAn open challenge to advance probabilistic forecasting for dengue epidemics\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EProceedings of the National Academy of Sciences\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E116\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E48\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E24268\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E74\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1073\/pnas.1909865116\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DProceedings%2Bof%2Bthe%2BNational%2BAcademy%2Bof%2BSciences%26rft_id%253Dinfo%253Adoi%252F10.1073%252Fpnas.1909865116%26rft_id%253Dinfo%253Apmid%252F31712420%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/ijlink\/YTozOntzOjQ6InBhdGgiO3M6MTQ6Ii9sb29rdXAvaWpsaW5rIjtzOjU6InF1ZXJ5IjthOjQ6e3M6ODoibGlua1R5cGUiO3M6NDoiQUJTVCI7czoxMToiam91cm5hbENvZGUiO3M6NDoicG5hcyI7czo1OiJyZXNpZCI7czoxMjoiMTE2LzQ4LzI0MjY4IjtzOjQ6ImF0b20iO3M6NTA6Ii9tZWRyeGl2L2Vhcmx5LzIwMjQvMDYvMjUvMjAyNC4wNi4yNC4yNDMwOTQxNi5hdG9tIjt9czo4OiJmcmFnbWVudCI7czowOiIiO30=\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-ijlink\u0022\u003E\u003Cspan\u003E\u003Cspan class=\u0022cit-reflinks-abstract\u0022\u003EAbstract\u003C\/span\u003E\u003Cspan class=\u0022cit-sep cit-reflinks-variant-name-sep\u0022\u003E\/\u003C\/span\u003E\u003Cspan class=\u0022cit-reflinks-full-text\u0022\u003E\u003Cspan class=\u0022free-full-text\u0022\u003EFREE \u003C\/span\u003EFull Text\u003C\/span\u003E\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-15-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-15\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.15\u0022 data-doi=\u002210.1287\/mnsc.1120.1667\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ELichtendahl\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKenneth C.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EYael\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EGrushka-Cockayne\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERobert L.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EWinkler\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2013\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EIs It Better to Average Probabilities or Quantiles?\u003C\/span\u003E\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EManagement Science\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E59\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E7\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E1594\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E1611\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1287\/mnsc.1120.1667\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DManagement%2BScience%26rft_id%253Dinfo%253Adoi%252F10.1287%252Fmnsc.1120.1667%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1287\/mnsc.1120.1667\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-16-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-16\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.16\u0022 data-doi=\u002210.1038\/s41598-018-36361-9\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EMcGowan\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ECraig J.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMatthew\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBiggerstaff\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMichael\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EJohansson\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKaryn M.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EApfeldorf\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMichal\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBen-Nun\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ELogan\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBrooks\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMatteo\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EConvertino\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2019\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003ECollaborative Efforts to Forecast Seasonal Influenza in the United States, 2015\u20132016\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EScientific Reports\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E9\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E1\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E683\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1038\/s41598-018-36361-9\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DScientific%2BReports%26rft.volume%253D9%26rft.spage%253D683%26rft_id%253Dinfo%253Adoi%252F10.1038%252Fs41598-018-36361-9%26rft_id%253Dinfo%253Apmid%252F30679458%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1038\/s41598-018-36361-9\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=30679458\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F25%2F2024.06.24.24309416.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-17-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-17\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.17\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ENiederreiter\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EHarald\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E1992\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003ERandom Number Generation and Quasi-Monte Carlo Methods\u003C\/span\u003E. \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EPhiladelphia PA: Society for Industrial; Applied Mathematics\u003C\/abbr\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-18-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-18\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.18\u0022 data-doi=\u002210.1073\/pnas.2103302119\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EPaireau\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJuliette\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAlessio\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EAndronico\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ENathana\u00ebl\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EHoz\u00e9\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMaylis\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ELayan\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EPascal\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ECr\u00e9pey\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAlix\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ERoumagnac\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMarc\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ELavielle\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EPierre-Yves\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBo\u00eblle\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ESimon\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ECauchemez\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2022\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EAn Ensemble Model Based on Early Predictors to Forecast COVID-19 Health Care Demand in France\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EProceedings of the National Academy of Sciences\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E119\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E18\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003Ee2103302119\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1073\/pnas.2103302119\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DProceedings%2Bof%2Bthe%2BNational%2BAcademy%2Bof%2BSciences%26rft.volume%253D119%26rft.spage%253De2103302119%26rft_id%253Dinfo%253Adoi%252F10.1073%252Fpnas.2103302119%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1073\/pnas.2103302119\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-19-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-19\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.19\u0022 data-doi=\u002210.5555\/1953048.2078195\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EPedregosa\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EFabian\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EGa\u00ebl\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EVaroquaux\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAlexandre\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EGramfort\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EVincent\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EMichel\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EBertrand\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EThirion\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EOlivier\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EGrisel\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMathieu\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBlondel\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2011\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EScikit-Learn: Machine Learning in Python\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJournal of Machine Learning Research\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E12\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E85\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E2825\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E30\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.5555\/1953048.2078195\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJournal%2Bof%2BMachine%2BLearning%2BResearch%26rft_id%253Dinfo%253Adoi%252F10.5555%252F1953048.2078195%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.5555\/1953048.2078195\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-20-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-20\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.20\u0022 data-doi=\u002210.1016\/j.ijforecast.2022.06.005\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ERay\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEvan L.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ELogan C.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBrooks\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJacob\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBien\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMatthew\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBiggerstaff\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ENikos I.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBosse\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJohannes\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EBracher\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEstee Y.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ECramer\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2023\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EComparing Trained and Untrained Probabilistic Ensemble Forecasts of COVID-19 Cases and Deaths in the United States\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EInternational Journal of Forecasting\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E39\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E3\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E1366\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E83\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1016\/j.ijforecast.2022.06.005\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DInternational%2BJournal%2Bof%2BForecasting%26rft_id%253Dinfo%253Adoi%252F10.1016%252Fj.ijforecast.2022.06.005%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1016\/j.ijforecast.2022.06.005\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-21-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-21\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.21\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ERay\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEvan L.\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAaron\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EGerding\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2024\u003C\/span\u003E. \u003Cspan class=\u0022cit-article-title\u0022\u003EDistfromq: Reconstruct a Distribution from a Collection of Quantiles\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-22-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-22\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.22\u0022 data-doi=\u002210.1371\/journal.pcbi.1005910\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ERay\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEvan L.\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ENicholas G.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EReich\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2018\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EPrediction of Infectious Disease Epidemics via Weighted Density Ensembles\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EPLOS Computational Biology\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E14\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E2\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003Ee1005910\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1371\/journal.pcbi.1005910\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DPLOS%2BComputational%2BBiology%26rft.volume%253D14%26rft.spage%253De1005910%26rft_id%253Dinfo%253Adoi%252F10.1371%252Fjournal.pcbi.1005910%26rft_id%253Dinfo%253Apmid%252F29462167%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1371\/journal.pcbi.1005910\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=29462167\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F25%2F2024.06.24.24309416.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-23-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-23\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.23\u0022 data-doi=\u002210.1038\/s41597-021-00839-5\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EReich\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ENicholas G.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMatthew\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ECornell\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEvan L.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ERay\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKatie\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EHouse\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKhoa\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ELe\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2021\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EThe Zoltar Forecast Archive, a Tool to Standardize and Store Interdisciplinary Prediction Research\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EScientific Data\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E8\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E1\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E59\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1038\/s41597-021-00839-5\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DScientific%2BData%26rft.volume%253D8%26rft.spage%253D59%26rft_id%253Dinfo%253Adoi%252F10.1038%252Fs41597-021-00839-5%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1038\/s41597-021-00839-5\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-24-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-24\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.24\u0022 data-doi=\u002210.2105\/AJPH.2022.306831\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EReich\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ENicholas G.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJustin\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ELessler\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ESebastian\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EFunk\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ECecile\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EViboud\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAlessandro\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EVespignani\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERyan J.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ETibshirani\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKatriona\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EShea\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2022\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003ECollaborative Hubs: Making the Most of Predictive Epidemic Modeling\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EAmerican Journal of Public Health\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E112\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E6\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E839\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E42\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.2105\/AJPH.2022.306831\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DAmerican%2BJournal%2Bof%2BPublic%2BHealth%26rft.volume%253D112%26rft.spage%253D839%26rft_id%253Dinfo%253Adoi%252F10.2105%252FAJPH.2022.306831%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.2105\/AJPH.2022.306831\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-25-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-25\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.25\u0022 data-doi=\u002210.1371\/journal.pcbi.1007486\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EReich\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ENicholas G.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ECraig J.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EMcGowan\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ETeresa K.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EYamana\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAbhinav\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ETushar\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EEvan L.\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ERay\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EDave\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EOsthus\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ESasikiran\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EKandula\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-etal\u0022\u003Eet al.\u003C\/span\u003E \u003Cspan class=\u0022cit-pub-date\u0022\u003E2019\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EAccuracy of Real-Time Multi-Model Ensemble Forecasts for Seasonal Influenza in the U.S\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EPLOS Computational Biology\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E15\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E11\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003Ee1007486\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1371\/journal.pcbi.1007486\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DPLOS%2BComputational%2BBiology%26rft.volume%253D15%26rft.spage%253D1007486e%26rft_id%253Dinfo%253Adoi%252F10.1371%252Fjournal.pcbi.1007486%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1371\/journal.pcbi.1007486\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-26-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-26\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.26\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EStone\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EM\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E1961\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EThe Opinion Pool\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EThe Annals of Mathematical Statistics\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E32\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E4\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E1339\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E42\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DThe%2BAnnals%2Bof%2BMathematical%2BStatistics%26rft.volume%253D32%26rft.spage%253D1339%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-27-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-27\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.27\u0022 data-doi=\u002210.1098\/rsta.2007.2076\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ETebaldi\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EClaudia\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EReto\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EKnutti\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2007\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EThe Use of the Multi-Model Ensemble in Probabilistic Climate Projections\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EPhilosophical Transactions: Mathematical, Physical and Engineering Sciences\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E365\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E1857\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E2053\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E75\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1098\/rsta.2007.2076\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DPhilosophical%2BTransactions%2Bof%2Bthe%2BRoyal%2BSociety%2BA%253A%2BMathematical%252C%2BPhysical%2Band%2BEngineering%2BSciences%26rft.stitle%253DPhil%2BTrans%2BR%2BSoc%2BA%26rft.aulast%253DTebaldi%26rft.auinit1%253DC.%26rft.volume%253D365%26rft.issue%253D1857%26rft.spage%253D2053%26rft.epage%253D2075%26rft.atitle%253DThe%2Buse%2Bof%2Bthe%2Bmulti-model%2Bensemble%2Bin%2Bprobabilistic%2Bclimate%2Bprojections%26rft_id%253Dinfo%253Adoi%252F10.1098%252Frsta.2007.2076%26rft_id%253Dinfo%253Apmid%252F17569654%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1098\/rsta.2007.2076\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=17569654\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F25%2F2024.06.24.24309416.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=000247573300005\u0026amp;link_type=ISI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-newisilink cit-ref-sprinkles-webofscience\u0022\u003E\u003Cspan\u003EWeb of Science\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-28-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-28\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-book\u0022 id=\u0022cit-2024.06.24.24309416v1.28\u0022 data-doi=\u002210.1016\/S1574-0706(05)01004-9\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003ETimmermann\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAllan\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2006\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-chapter-title\u0022\u003EChapter 4 Forecast Combinations\u003C\/span\u003E.\u201d In, \u003Cspan class=\u0022cit-vol\u0022\u003E1\u003C\/span\u003E:\u003Cspan class=\u0022cit-fpage\u0022\u003E135\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E96\u003C\/span\u003E. \u003Cspan class=\u0022cit-publ-name\u0022\u003EElsevier\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1016\/S1574-0706(05)01004-9\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DHANDBOOK%2BOF%2BECONOMIC%2BFORECASTING%26rft.volume%253D1%26rft.spage%253D135%26rft_id%253Dinfo%253Adoi%252F10.1016%252FS1574-0706%252805%252901004-9%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1016\/S1574-0706(05)01004-9\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-29-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-29\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-book\u0022 id=\u0022cit-2024.06.24.24309416v1.29\u0022 data-doi=\u002210.1016\/J.EPIDEM.2017.08.002.\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EViboud\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EC\u00e9cile\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EKaiyuan\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ESun\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERobert\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EGaffey\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EMarco\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EAjelli\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ELaura\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EFumanelli\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EStefano\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EMerler\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EQian\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EZhang\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EGerardo\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EChowell\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ELone\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003ESimonsen\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EAlessandro\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EVespignani\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2018\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-chapter-title\u0022\u003EThe RAPIDD Ebola Forecasting Challenge: Synthesis and Lessons Learnt\u003C\/span\u003E.\u201d \u003Cspan class=\u0022cit-source\u0022\u003EEpidemics\u003C\/span\u003E, \u003Cspan class=\u0022cit-publ-name\u0022\u003EThe RAPIDD Ebola Forecasting Challenge\u003C\/span\u003E, \u003Cspan class=\u0022cit-vol\u0022\u003E22\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003EMarch\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E13\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E21\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1016\/j.epidem.2017.08.002\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DEpidemics%26rft.volume%253D22%26rft.spage%253D13%26rft_id%253Dinfo%253Adoi%252F10.1016%252FJ.EPIDEM.2017.08.002.%26rft_id%253Dinfo%253Apmid%252F28958414%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1016\/J.EPIDEM.2017.08.002.\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=28958414\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F25%2F2024.06.24.24309416.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-30-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-30\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-thesis\u0022 id=\u0022cit-2024.06.24.24309416v1.30\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EVincent\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003EStella Burnham\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E1912\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-chapter-title\u0022\u003EThe Function of the Vibrissae in the Behavior of the White Rat\u003C\/span\u003E.\u201d PhD thesis, \u003Cspan class=\u0022cit-publ-loc\u0022\u003ECambridge MA:\u003C\/span\u003E \u003Cspan class=\u0022cit-publ-name\u0022\u003EUniversity of Chicago\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal no-rev-xref\u0022 id=\u0022cit-2024.06.24.24309416v1.31\u0022 data-doi=\u002210.32614\/RJ-2018-052\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth cit-collab\u0022\u003EWeiss, Christoph,E., Eran Raviv, and Gernot Roetzer\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2019\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EForecast Combinations in R Using the ForecastComb Package\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EThe R Journal\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E10\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E2\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E262\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.32614\/RJ-2018-052\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DThe%2BR%2BJournal%26rft.volume%253D10%26rft.spage%253D262%26rft_id%253Dinfo%253Adoi%252F10.32614%252FRJ-2018-052%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.32614\/RJ-2018-052\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-32-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-32\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.32\u0022 data-doi=\u002210.1111\/risa.12302\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EWinkler\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ERobert L\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2015\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003EEqual Versus Differential Weighting in Combining Forecasts\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003ERisk Analysis\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E35\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E1\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E16\u003C\/span\u003E\u2013\u003Cspan class=\u0022cit-lpage\u0022\u003E18\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1111\/risa.12302\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DRisk%2BAnalysis%26rft.volume%253D35%26rft.spage%253D16%26rft_id%253Dinfo%253Adoi%252F10.1111%252Frisa.12302%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1111\/risa.12302\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003Cli\u003E\u003Cspan class=\u0022ref-label ref-label-empty\u0022\u003E\u003C\/span\u003E\u003Ca class=\u0022rev-xref-ref\u0022 href=\u0022#xref-ref-33-1\u0022 title=\u0022View reference  in text\u0022 id=\u0022ref-33\u0022\u003E\u21b5\u003C\/a\u003E\u003Cdiv class=\u0022cit ref-cit ref-journal\u0022 id=\u0022cit-2024.06.24.24309416v1.33\u0022 data-doi=\u002210.1098\/rsif.2016.0410\u0022\u003E\u003Cdiv class=\u0022cit-metadata\u0022\u003E\u003Ccite\u003E\u003Cspan class=\u0022cit-auth\u0022\u003E\u003Cspan class=\u0022cit-name-surname\u0022\u003EYamana\u003C\/span\u003E,  \u003Cspan class=\u0022cit-name-given-names\u0022\u003ETeresa K.\u003C\/span\u003E\u003C\/span\u003E, \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003ESasikiran\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EKandula\u003C\/span\u003E\u003C\/span\u003E, and \u003Cspan class=\u0022cit-auth\u0022\u003E \u003Cspan class=\u0022cit-name-given-names\u0022\u003EJeffrey\u003C\/span\u003E \u003Cspan class=\u0022cit-name-surname\u0022\u003EShaman\u003C\/span\u003E\u003C\/span\u003E. \u003Cspan class=\u0022cit-pub-date\u0022\u003E2016\u003C\/span\u003E. \u201c\u003Cspan class=\u0022cit-article-title\u0022\u003ESuperensemble Forecasts of Dengue Outbreaks\u003C\/span\u003E.\u201d \u003Cabbr class=\u0022cit-jnl-abbrev\u0022\u003EJournal of The Royal Society Interface\u003C\/abbr\u003E \u003Cspan class=\u0022cit-vol\u0022\u003E13\u003C\/span\u003E (\u003Cspan class=\u0022cit-issue\u0022\u003E123\u003C\/span\u003E): \u003Cspan class=\u0022cit-fpage\u0022\u003E20160410\u003C\/span\u003E.\u003Cspan class=\u0022cit-pub-id-sep cit-pub-id-doi-sep\u0022\u003E \u003C\/span\u003E\u003Cspan class=\u0022cit-pub-id cit-pub-id-doi\u0022\u003E\u003Cspan class=\u0022cit-pub-id-scheme-doi\u0022\u003Edoi:\u003C\/span\u003E10.1098\/rsif.2016.0410\u003C\/span\u003E.\u003C\/cite\u003E\u003C\/div\u003E\u003Cdiv class=\u0022cit-extra\u0022\u003E\u003Ca href=\u0022{openurl}?query=rft.jtitle%253DJournal%2Bof%2BThe%2BRoyal%2BSociety%2BInterface%26rft_id%253Dinfo%253Adoi%252F10.1098%252Frsif.2016.0410%26rft_id%253Dinfo%253Apmid%252F27733698%26rft.genre%253Darticle%26rft_val_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Ajournal%26ctx_ver%253DZ39.88-2004%26url_ver%253DZ39.88-2004%26url_ctx_fmt%253Dinfo%253Aofi%252Ffmt%253Akev%253Amtx%253Actx\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-openurl cit-ref-sprinkles-open-url\u0022\u003E\u003Cspan\u003EOpenUrl\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=10.1098\/rsif.2016.0410\u0026amp;link_type=DOI\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-doi cit-ref-sprinkles-crossref\u0022\u003E\u003Cspan\u003ECrossRef\u003C\/span\u003E\u003C\/a\u003E\u003Ca href=\u0022\/lookup\/external-ref?access_num=27733698\u0026amp;link_type=MED\u0026amp;atom=%2Fmedrxiv%2Fearly%2F2024%2F06%2F25%2F2024.06.24.24309416.atom\u0022 class=\u0022cit-ref-sprinkles cit-ref-sprinkles-medline\u0022\u003E\u003Cspan\u003EPubMed\u003C\/span\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003C\/li\u003E\u003C\/ol\u003E\u003C\/div\u003E\u003Cspan class=\u0022highwire-journal-article-marker-end\u0022\u003E\u003C\/span\u003E\u003C\/div\u003E\u003Cspan class=\u0022related-urls\u0022\u003E\u003C\/span\u003E\u003C\/div\u003E\u003C\/div\u003E  \u003C\/div\u003E\n\n  \n  \u003C\/div\u003E\n\u003C\/div\u003E\n  \u003C\/div\u003E\n\u003C\/div\u003E\n\u003C\/div\u003E\u003Cscript type=\u0022text\/javascript\u0022 src=\u0022https:\/\/www.medrxiv.org\/sites\/default\/files\/js\/js_zP7WWIfzbyzvaM63L39cNV2juU_1XVH7wduFK9gcMNI.js\u0022\u003E\u003C\/script\u003E\n\u003C\/body\u003E\u003C\/html\u003E"}