spb/wp20_uqo

1{"status": "ok", "message-type": "work-list", "message-version": "1.0.0", "message": {"facets": {}, "total-results": 15188203, "items": [{"indexed": {"date-parts": [[2026, 10, 20]], "date-time": "2026-10-20T02:11:27Z", "timestamp": 1792462287086, "version": "4.5.0"}, "publisher-location": "San Diego, California, USA", "reference-count": 0, "publisher": "Neural Information Processing Systems Foundation, Inc. (NeurIPS)", "content-domain": {"domain": [], "crossmark-restriction": false}, "published-print": {"date-parts": [[2023]]}, "DOI": "10.52202/075280-1516", "type": "proceedings-article", "created": {"date-parts": [[2026, 3, 2]], "date-time": "2026-03-02T13:18:04Z", "timestamp": 1772457484000}, "page": "34892-34916", "source": "Crossref", "is-referenced-by-count": 2030, "title": ["Visual Instruction Tuning"], "prefix": "10.52202", "author": [{"given": "Haotian", "family": "Liu", "sequence": "first", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Chunyuan", "family": "Li", "sequence": "additional", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Qingyang", "family": "Wu", "sequence": "additional", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Yong Jae", "family": "Lee", "sequence": "additional", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}], "member": "29733", "event": {"name": "Advances in Neural Information Processing Systems 36", "theme": "37th Conference on Neural Information Processing Systems (NeurIPS 2023)", "location": "New Orleans, Louisiana, USA", "acronym": "NeurIPS 2023", "number": "36", "start": {"date-parts": [[2023, 12, 10]]}, "end": {"date-parts": [[2023, 12, 16]]}}, "container-title": ["Advances in Neural Information Processing Systems 36"], "deposited": {"date-parts": [[2026, 6, 19]], "date-time": "2026-06-19T15:37:57Z", "timestamp": 1781883477000}, "score": 31.40541, "resource": {"primary": {"URL": "http://www.proceedings.com/075280-1516.html"}}, "issued": {"date-parts": [[2023]]}, "references-count": 0, "URL": "https://doi.org/10.52202/075280-1516", "published": {"date-parts": [[2023]]}}, {"indexed": {"date-parts": [[2026, 10, 20]], "date-time": "2026-10-20T02:11:10Z", "timestamp": 1792462270467, "version": "4.5.0"}, "reference-count": 60, "publisher": "IEEE", "license": [{"start": {"date-parts": [[2024, 6, 16]], "date-time": "2024-06-16T00:00:00Z", "timestamp": 1718496000000}, "content-version": "stm-asf", "delay-in-days": 0, "URL": "https://doi.org/10.15223/policy-029"}, {"start": {"date-parts": [[2024, 6, 16]], "date-time": "2024-06-16T00:00:00Z", "timestamp": 1718496000000}, "content-version": "stm-asf", "delay-in-days": 0, "URL": "https://doi.org/10.15223/policy-037"}], "content-domain": {"domain": [], "crossmark-restriction": false}, "published-print": {"date-parts": [[2024, 6, 16]]}, "DOI": "10.1109/cvpr52733.2024.02484", "type": "proceedings-article", "created": {"date-parts": [[2024, 9, 16]], "date-time": "2024-09-16T13:34:53Z", "timestamp": 1726493693000}, "page": "26286-26296", "source": "Crossref", "is-referenced-by-count": 1852, "title": ["Improved Baselines with Visual Instruction Tuning"], "prefix": "10.1109", "author": [{"given": "Haotian", "family": "Liu", "sequence": "first", "affiliation": [{"name": "University of Wisconsin,Madison"}], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Chunyuan", "family": "Li", "sequence": "additional", "affiliation": [{"name": "Microsoft Research,Redmond"}], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Yuheng", "family": "Li", "sequence": "additional", "affiliation": [{"name": "University of Wisconsin,Madison"}], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Yong Jae", "family": "Lee", "sequence": "additional", "affiliation": [{"name": "University of Wisconsin,Madison"}], "role": [{"vocabulary": "crossref", "role": "author"}]}], "member": "263", "reference": [{"key": "ref1", "volume-title": "Fuyu-8b: A multimodal architecture for ai agents.", "year": "2024"}, {"key": "ref2", "author": "Alayrac", "year": "2022", "journal-title": "Flamingo: a vi-suallanguage model for few-shot learning."}, {"key": "ref3", "author": "Bai", "year": "2023", "journal-title": "Qwen-vl: A frontier large vision-language model with versatile abilities."}, {"key": "ref4", "author": "Bitton", "year": "2023", "journal-title": "Visit-bench: A benchmark for vision-language instruction following inspired by real-world use"}, {"key": "ref5", "author": "Black", "year": "2023", "journal-title": "Training diffusion models with reinforcement learning."}, {"key": "ref6", "author": "Carlini", "year": "2023", "journal-title": "Are aligned neural networks adversarially aligned?"}, {"key": "ref7", "doi-asserted-by": "publisher", "DOI": "10.1609/aaai.v38i16.29727"}, {"key": "ref8", "author": "Chen", "year": "2023", "journal-title": "Shikra: Unleashing multimodalllms referential dialogue magic."}, {"key": "ref9", "article-title": "A simple framework for contrastive learning of visual representations", "volume-title": "ICML", "author": "Chen", "year": "2020"}, {"key": "ref10", "author": "Chen", "year": "2020", "journal-title": "Improved baselines with momentum contrastive learning."}, {"key": "ref11", "author": "Chen", "year": "2023", "journal-title": "Pali-x: On scaling up a multilingual vision and language model."}, {"key": "ref12", "author": "Chung", "year": "2022", "journal-title": "Scaling instruction-finetuned language models."}, {"key": "ref13", "author": "Dai", "year": "2023", "journal-title": "Instructblip: Towards general-purpose vision-language models with instruction tuning."}, {"key": "ref14", "author": "Dosovitskiy", "year": "2020", "journal-title": "An image is worth 16x16 words: Trans-formers for image recognition at scale."}, {"key": "ref15", "doi-asserted-by": "publisher", "DOI": "10.1109/CVPR52729.2023.01855"}, {"key": "ref16", "author": "Fu", "year": "2023", "journal-title": "Mme: A comprehensive evaluation benchmark for multimodallarge language models."}, {"key": "ref17", "author": "Gong", "year": "2023", "journal-title": "Multimodal-gpt: A vision and language model for dialogue with humans."}, {"key": "ref18", "doi-asserted-by": "publisher", "DOI": "10.1109/CVPR.2017.670"}, {"key": "ref19", "doi-asserted-by": "publisher", "DOI": "10.1109/CVPR.2018.00380"}, {"key": "ref20", "doi-asserted-by": "publisher", "DOI": "10.1109/CVPR.2019.00686"}, {"key": "ref21", "volume-title": "Introducing idefics: An open reproduction of state-of-the-art visual language model.", "year": "2023"}, {"key": "ref22", "article-title": "Openclip", "author": "Ilharco", "journal-title": "2021 If you use this software, please cite it as below."}, {"key": "ref23", "doi-asserted-by": "publisher", "DOI": "10.3115/v1/D14-1086"}, {"key": "ref24", "doi-asserted-by": "publisher", "DOI": "10.1007/s11263-016-0981-7"}, {"key": "ref25", "author": "Lai", "year": "2023", "journal-title": "Lisa: Reasoning segmentation via large language model."}, {"key": "ref26", "author": "Li", "year": "2023", "journal-title": "Seed-bench: Benchmarking mul-timodal llms with generative comprehension."}, {"key": "ref27", "author": "Li", "year": "2023", "journal-title": "Otterhd: A high-resolution multi-modality model"}, {"key": "ref28", "author": "Li", "year": "2023", "journal-title": "Mimic-it: Multi-modal in-context instruction tuning."}, {"key": "ref29", "author": "Li", "year": "2023", "journal-title": "Multimodal foundation models: From specialists to general-purpose assistants."}, {"key": "ref30", "author": "Li", "year": "2023", "journal-title": "Llava-med: Training a large language-and-vision assistant for biomedicine in one day."}, {"key": "ref31", "author": "Li", "year": "2023", "journal-title": "Blip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models."}, {"key": "ref32", "doi-asserted-by": "publisher", "DOI": "10.18653/v1/2021.acl-long.353"}, {"key": "ref33", "doi-asserted-by": "publisher", "DOI": "10.18653/v1/2023.emnlp-main.20"}, {"key": "ref34", "doi-asserted-by": "publisher", "DOI": "10.1007/978-3-319-10602-1_48"}, {"key": "ref35", "article-title": "Visual instruction tuning", "volume-title": "NeurIPS", "author": "Liu", "year": "2023"}, {"key": "ref36", "author": "Liu", "year": "2023", "journal-title": "Mmbench: Is your multi-modal model an all-around player?"}, {"key": "ref37", "article-title": "Learn to explain: Multimodal reasoning via thought chains for science question answering", "author": "Lu", "year": "2022", "journal-title": "Advances in Neural Information Processing Systems"}, {"key": "ref38", "author": "Lu", "year": "2023", "journal-title": "An empirical study of scaling instruct-tuned large multimodal models."}, {"key": "ref39", "doi-asserted-by": "publisher", "DOI": "10.1109/CVPR.2016.9"}, {"key": "ref40", "doi-asserted-by": "publisher", "DOI": "10.1109/CVPR.2019.00331"}, {"key": "ref41", "doi-asserted-by": "publisher", "DOI": "10.1109/ICDAR.2019.00156"}, {"key": "ref42", "volume-title": "Gpt-4v(ision) system card.", "year": "2023"}, {"key": "ref43", "author": "Radford", "year": "2021", "journal-title": "Learning transferable visual models from natural language supervision."}, {"key": "ref44", "first-page": "146", "article-title": "A-okvqa: A benchmark for visual question answering using world know ledge", "volume-title": "European Conference on Computer Vision", "author": "Schwenk", "year": "2022"}, {"key": "ref45", "volume-title": "ShareGPT.", "year": "2023"}, {"key": "ref46", "doi-asserted-by": "publisher", "DOI": "10.1007/978-3-030-58536-5_44"}, {"key": "ref47", "doi-asserted-by": "publisher", "DOI": "10.1109/CVPR.2019.00851"}, {"key": "ref48", "author": "Touvron", "year": "2023", "journal-title": "Llama 2: Open foundation and fine-tuned chat models."}, {"key": "ref49", "author": "Wang", "year": "2023", "journal-title": "Visionllm: Large language model is also an open-ended decoder for vision-centric tasks."}, {"key": "ref50", "author": "Wei", "year": "2021", "journal-title": "Finetuned language models are zero-shot learners."}, {"key": "ref51", "author": "Yang", "year": "2023", "journal-title": "The dawn of lmms: Preliminary explorations with gpt-4v (ision)."}, {"key": "ref52", "doi-asserted-by": "publisher", "DOI": "10.18653/v1/2023.findings-emnlp.187"}, {"key": "ref53", "author": "Ye", "year": "2023", "journal-title": "mplugow 1: Modularization empowers large language models with multimodality."}, {"key": "ref54", "author": "Yu", "year": "2023", "journal-title": "Mm-vet: Evaluating large multimodal models for integrated capabilities."}, {"key": "ref55", "author": "Zhang", "year": "2023", "journal-title": "Gpt4roi: Instruction tuning large language model on region-of-interest."}, {"key": "ref56", "author": "Zhang", "year": "2023", "journal-title": "Llavar: Enhanced visual instruction tuning for textrich image understanding."}, {"key": "ref57", "author": "Zhao", "year": "2023", "journal-title": "Svit: Scaling up visual instruction tuning."}, {"key": "ref58", "author": "Zhao", "year": "2023", "journal-title": "On evaluating adversarial robustness of large vision-language models."}, {"key": "ref59", "author": "Zhou", "year": "2023", "journal-title": "Lima: Less is more for alignment."}, {"key": "ref60", "author": "Zhu", "year": "2023", "journal-title": "Minigpt-4: Enhancing vision-language understanding with advanced large language models."}], "event": {"name": "2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "location": "Seattle, WA, USA", "start": {"date-parts": [[2024, 6, 16]]}, "end": {"date-parts": [[2024, 6, 22]]}}, "container-title": ["2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)"], "link": [{"URL": "http://xplorestaging.ieee.org/ielx8/10654794/10654797/10655294.pdf?arnumber=10655294", "content-type": "unspecified", "content-version": "vor", "intended-application": "similarity-checking"}], "deposited": {"date-parts": [[2024, 9, 20]], "date-time": "2024-09-20T01:32:21Z", "timestamp": 1726795941000}, "score": 25.512218, "resource": {"primary": {"URL": "https://ieeexplore.ieee.org/document/10655294/"}}, "issued": {"date-parts": [[2024, 6, 16]]}, "references-count": 60, "URL": "https://doi.org/10.1109/cvpr52733.2024.02484", "published": {"date-parts": [[2024, 6, 16]]}}, {"indexed": {"date-parts": [[2026, 10, 13]], "date-time": "2026-10-13T18:03:41Z", "timestamp": 1791914621196, "version": "4.3.4"}, "reference-count": 0, "publisher": "Association for the Advancement of Artificial Intelligence (AAAI)", "issue": "16", "content-domain": {"domain": [], "crossmark-restriction": false}, "short-container-title": ["AAAI"], "abstract": "<jats:p>Recent research has demonstrated that the multi-task fine-tuning of multi-modal Large Language Models (LLMs) using an assortment of annotated downstream vision-language datasets significantly enhances their performance. Yet, during this process, a side effect, which we termed as the \"multi-modal alignment tax\", surfaces. This side effect negatively impacts the model's ability to format responses appropriately - for instance, its \"politeness\" - due to the overly succinct and unformatted nature of raw annotations, resulting in reduced human preference. In this paper, we introduce Polite Flamingo, a multi-modal response rewriter that transforms raw annotations into a more appealing, \"polite\" format. Polite Flamingo is trained to reconstruct high-quality responses from their automatically distorted counterparts and is subsequently applied to a vast array of vision-language datasets for response rewriting. After rigorous filtering, we generate the PF-1M dataset and further validate its value by fine-tuning a multi-modal LLM with it. Combined with novel methodologies including U-shaped multi-stage tuning and multi-turn augmentation, the resulting model, Clever Flamingo, demonstrates its advantages in both multi-modal understanding and response politeness according to automated and human evaluations. Code and dataset are available at https://github.com/ChenDelong1999/polite-flamingo</jats:p>", "DOI": "10.1609/aaai.v38i16.29727", "type": "journal-article", "created": {"date-parts": [[2024, 3, 25]], "date-time": "2024-03-25T07:51:10Z", "timestamp": 1711353070000}, "page": "17745-17753", "source": "Crossref", "is-referenced-by-count": 14, "title": ["Visual Instruction Tuning with Polite Flamingo"], "prefix": "10.1609", "volume": "38", "author": [{"given": "Delong", "family": "Chen", "sequence": "first", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Jianfeng", "family": "Liu", "sequence": "additional", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Wenliang", "family": "Dai", "sequence": "additional", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Baoyuan", "family": "Wang", "sequence": "additional", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}], "member": "9382", "published-online": {"date-parts": [[2024, 3, 24]]}, "container-title": ["Proceedings of the AAAI Conference on Artificial Intelligence"], "link": [{"URL": "https://ojs.aaai.org/index.php/AAAI/article/download/29727/31249", "content-type": "application/pdf", "content-version": "vor", "intended-application": "text-mining"}, {"URL": "https://ojs.aaai.org/index.php/AAAI/article/download/29727/31249", "content-type": "unspecified", "content-version": "vor", "intended-application": "similarity-checking"}], "deposited": {"date-parts": [[2024, 3, 25]], "date-time": "2024-03-25T07:51:10Z", "timestamp": 1711353070000}, "score": 24.574203, "resource": {"primary": {"URL": "https://ojs.aaai.org/index.php/AAAI/article/view/29727"}}, "issued": {"date-parts": [[2024, 3, 24]]}, "references-count": 0, "journal-issue": {"issue": "16", "published-online": {"date-parts": [[2024, 3, 25]]}}, "URL": "https://doi.org/10.1609/aaai.v38i16.29727", "ISSN": ["2374-3468", "2159-5399"], "issn-type": [{"value": "2374-3468", "type": "electronic"}, {"value": "2159-5399", "type": "print"}], "published": {"date-parts": [[2024, 3, 24]]}}, {"indexed": {"date-parts": [[2026, 10, 17]], "date-time": "2026-10-17T23:23:53Z", "timestamp": 1792279433171, "version": "4.4.0"}, "posted": {"date-parts": [[2023]]}, "group-title": "SSRN", "reference-count": 9, "publisher": "Elsevier BV", "content-domain": {"domain": [], "crossmark-restriction": false}, "abstract": "<jats:p>Sentiment analysis is a vital tool for uncovering insights from financial articles, news, and social media, shaping our understanding of market movements. Despite the impressive capabilities of large language models (LLMs) in financial natural language processing (NLP), they still struggle with accurately interpreting numerical values and grasping financial context, limiting their effectiveness in predicting financial sentiment. In this paper, we introduce a simple yet effective instruction tuning approach to address these issues. By transforming a small portion of supervised financial sentiment analysis data into instruction data and fine-tuning a general-purpose LLM with this method, we achieve remarkable advancements in financial sentiment analysis. In the experiment, our approach outperforms state-of-the-art supervised sentiment analysis models, as well as widely used LLMs like ChatGPT and LLaMAs, particularly in scenarios where numerical understanding and contextual comprehension are vital.</jats:p>", "DOI": "10.2139/ssrn.4489831", "type": "posted-content", "created": {"date-parts": [[2023, 7, 3]], "date-time": "2023-07-03T13:15:12Z", "timestamp": 1688390112000}, "source": "Crossref", "is-referenced-by-count": 94, "title": ["Instruct-FinGPT: Financial Sentiment Analysis by Instruction Tuning of General-Purpose Large Language Models"], "prefix": "10.2139", "author": [{"given": "Boyu", "family": "Zhang", "sequence": "first", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Hongyang", "family": "Yang", "sequence": "additional", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Xiao-Yang", "family": "Liu", "sequence": "additional", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}], "member": "78", "reference": [{"key": "ref1", "author": "Araci", "year": "2019", "journal-title": "Dogu Araci. FinBERT: Financial sentiment analysis with pre-trained language models"}, {"issue": "2", "key": "ref2", "first-page": "53", "article-title": "Financial news predicts stock market volatility better than close price", "volume": "4", "author": "Atkins", "year": "2017", "journal-title": "The Journal of Finance and Data Science"}, {"key": "ref3", "first-page": "4171", "article-title": "BERT: Pre-training of deep bidirectional transformers for language understanding", "volume": "2016", "author": "Lee ; Min-Yuh Day", "year": "2016", "journal-title": "IEEE/ACM International Conference on Advances in Social Networks Analysis and Mining (ASONAM)"}, {"issue": "1", "key": "ref4", "doi-asserted-by": "crossref", "first-page": "35", "DOI": "10.1111/j.1540-6261.2010.01625.x", "article-title": "When is a liability not a liability? Textual analysis, dictionaries, and 10-Ks", "volume": "66", "author": "Mcdonald Loughran", "year": "2011", "journal-title": "The Journal of Finance"}, {"key": "ref5", "first-page": "131662", "article-title": "Good debt or bad debt: Detecting semantic orientations in economic texts", "volume": "65", "author": "Neural Magic", "year": "2014", "journal-title": "Pekka Malo, Ankur Sinha, Pekka Korhonen, Jyrki Wallenius, and Pyry Takala"}, {"key": "ref6", "first-page": "3505", "article-title": "Deepspeed: System optimizations enable training deep learning models with over 100 billion parameters", "author": "Rasley", "year": "2020", "journal-title": "Association for Computing Machinery, KDD '20"}, {"key": "ref7", "first-page": "1", "article-title": "When flue meets flang: Benchmarks and large pretrained language model for financial domain", "volume": "5", "author": "Shah", "year": "2018", "journal-title": "Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP)"}, {"key": "ref8", "first-page": "53", "article-title": "Automatic domain-specific sentiment lexicon generation with label propagation", "author": "Kao Tai", "year": "2013", "journal-title": "Proceedings of International Conference on Information Integration and Web-based Applications & Services"}, {"key": "ref9", "first-page": "49", "article-title": "Natural language based financial forecasting: a survey", "volume": "50", "author": "Wu", "year": "2018", "journal-title": "A large language model for finance"}], "deposited": {"date-parts": [[2026, 8, 14]], "date-time": "2026-08-14T15:56:14Z", "timestamp": 1786722974000}, "score": 24.342672, "resource": {"primary": {"URL": "https://www.ssrn.com/abstract=4489831"}}, "issued": {"date-parts": [[2023]]}, "references-count": 9, "URL": "https://doi.org/10.2139/ssrn.4489831", "published": {"date-parts": [[2023]]}, "subtype": "preprint"}, {"indexed": {"date-parts": [[2026, 10, 19]], "date-time": "2026-10-19T19:40:07Z", "timestamp": 1792438807874, "version": "4.5.0"}, "publisher-location": "San Diego, California, USA", "reference-count": 0, "publisher": "Neural Information Processing Systems Foundation, Inc. (NeurIPS)", "content-domain": {"domain": [], "crossmark-restriction": false}, "published-print": {"date-parts": [[2025]]}, "DOI": "10.52202/085713-4301", "type": "proceedings-article", "created": {"date-parts": [[2026, 8, 6]], "date-time": "2026-08-06T14:44:29Z", "timestamp": 1786027469000}, "page": "143258-143298", "source": "Crossref", "is-referenced-by-count": 2, "title": ["Visual Instruction Bottleneck Tuning"], "prefix": "10.52202", "author": [{"given": "Changdae", "family": "Oh", "sequence": "first", "affiliation": [{"name": "University of Wisconsin—Madison"}], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Jiatong", "family": "Li", "sequence": "additional", "affiliation": [{"name": "University of Wisconsin-Madison"}], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Shawn", "family": "Im", "sequence": "additional", "affiliation": [{"name": "UW-Madison"}], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Sharon", "family": "Li", "sequence": "additional", "affiliation": [{"name": "University of Wisconsin-Madison"}], "role": [{"vocabulary": "crossref", "role": "author"}]}], "member": "29733", "event": {"name": "Advances in Neural Information Processing Systems 38", "theme": "39th Conference on Neural Information Processing Systems (NeurIPS 2025)", "location": "San Diego, California, USA and Mexico City, Mexico", "acronym": "NeurIPS 2025", "number": "38", "start": {"date-parts": [[2025, 11, 30]]}, "end": {"date-parts": [[2025, 12, 7]]}}, "container-title": ["Advances in Neural Information Processing Systems 38"], "deposited": {"date-parts": [[2026, 10, 19]], "date-time": "2026-10-19T18:42:10Z", "timestamp": 1792435330000}, "score": 23.789984, "resource": {"primary": {"URL": "http://www.proceedings.com/085713-4301.html"}}, "issued": {"date-parts": [[2025]]}, "references-count": 0, "URL": "https://doi.org/10.52202/085713-4301", "published": {"date-parts": [[2025]]}}, {"institution": [{"id": [{"id": "https://ror.org/00fzmm222", "id-type": "ROR", "asserted-by": "publisher"}, {"id": "https://www.isni.org/0000000102217463", "id-type": "ISNI", "asserted-by": "publisher"}, {"id": "https://www.wikidata.org/entity/Q4118040", "id-type": "wikidata", "asserted-by": "publisher"}], "name": "University of Massachusetts Dartmouth", "place": ["Dartmouth, Massachusetts, United States"], "department": ["Department of Computer and Information Science"]}], "indexed": {"date-parts": [[2026, 7, 30]], "date-time": "2026-07-30T11:20:37Z", "timestamp": 1785410437782, "version": "3.56.0"}, "reference-count": 0, "publisher": "University of Massachusetts Dartmouth", "license": [{"content-version": "unspecified", "delay-in-days": 0, "URL": "http://creativecommons.org/licenses/by-nc-nd/4.0/"}], "content-domain": {"domain": [], "crossmark-restriction": false}, "chair": [{"given": "Ming Daniel", "family": "Shao", "sequence": "additional", "affiliation": [{"id": [{"id": "https://ror.org/03hamhx47", "id-type": "ROR", "asserted-by": "publisher"}, {"id": "https://www.isni.org/0000000096201122", "id-type": "ISNI", "asserted-by": "publisher"}, {"id": "https://www.wikidata.org/entity/Q15143", "id-type": "wikidata", "asserted-by": "publisher"}], "name": "University of Massachusetts Lowell (United States, Lowell)"}], "role": [{"vocabulary": "crossref", "role": "chair"}]}, {"given": "Jiawei", "family": "Yuan", "sequence": "additional", "affiliation": [{"id": [{"id": "https://ror.org/00fzmm222", "id-type": "ROR", "asserted-by": "publisher"}, {"id": "https://www.isni.org/0000000102217463", "id-type": "ISNI", "asserted-by": "publisher"}, {"id": "https://www.wikidata.org/entity/Q4118040", "id-type": "wikidata", "asserted-by": "publisher"}], "name": "University of Massachusetts Dartmouth"}], "role": [{"vocabulary": "crossref", "role": "chair"}]}, {"given": "Ashokkumar Ratilal", "family": "Patel", "sequence": "additional", "affiliation": [{"id": [{"id": "https://ror.org/00fzmm222", "id-type": "ROR", "asserted-by": "publisher"}, {"id": "https://www.isni.org/0000000102217463", "id-type": "ISNI", "asserted-by": "publisher"}, {"id": "https://www.wikidata.org/entity/Q4118040", "id-type": "wikidata", "asserted-by": "publisher"}], "name": "University of Massachusetts Dartmouth"}], "role": [{"vocabulary": "crossref", "role": "chair"}]}], "abstract": "<jats:p>Multimodal large language models (MLLMs), which can reason across different modalities, are opening up new frontiers in AI development that will soon enable complex reasoning with both visual and textual modalities. One application of multimodal MLLMs is Visual Question Answering (VQA) in healthcare; for example, a clinician can ask questions about medical images in natural language to receive detailed explanations. As suitably adapted for the modern age: \"The art of medicine consists in amusing the patient while nature cures the disease - but in the age of AI, assisting the doctor while learning from the data.\" - Inspired by Voltaire, adapted for modern AI healthcare This research examines the efficacy of fine-tuning multimodal (vision and language) foundational models for performing medical visual question answering. This study aims to evaluate how effectively these models can interpret and respond to medical queries based on visual inputs, ultimately enhancing diagnostic accuracy and patient care. By leveraging the strengths of both vision and language processing, the research seeks to advance the capabilities of AI in medical settings. The author studies three fine-tuned variants derived from the baseline LLaVA-Med architecture: a caption-only baseline trained to generate global descriptions of medical images; a model that uses instruction tuning to better answer diagnostic and region-specific questions, and a variant that incorporates Alpha-CLIP with LLaVA-Med to allow spatially targeted understanding by conditioning responses on user-defined regions of interest (ROIs). All variants are adapted utilizing Low-Rank Adaptation (LoRA), a parameter-efficient fine-tuning (PEFT) technique that facilitates minimal computational overhead. The results show that the foundational LLaVA-Med model, which was trained only on image-caption pairs, performs fairly well with BLEU (0.09) and ROUGE-2 (0.14), but it struggles to answer specific questions compared to the improved version that has the best ROUGE-1 (0.59) and ROUGE-L (0.54). The region-focused Alpha-CLIP combined with LLaVA-Med achieved the best performance on both the ROUGE-2 (0.40) and BLEU (0.28), while still exhibiting accurate and context-sensitive reasoning, which points to the importance of task-specific adaptation in medical VQA. The combined results suggest that concentrating attention on specific areas within images, along with instruction-based reasoning, is essential for medical AI systems that manage diverse types of data. This approach facilitates the comparison of methods and provides a detailed strategy for improving specialized diagnostic medical AI assistants.</jats:p>", "DOI": "10.62791/20485", "type": "dissertation", "created": {"date-parts": [[2025, 11, 12]], "date-time": "2025-11-12T21:26:08Z", "timestamp": 1762982768000}, "approved": {"date-parts": [[2025]]}, "source": "Crossref", "is-referenced-by-count": 0, "title": ["Fine-tuning multimodal large language models for medical visual question answering"], "prefix": "10.62791", "author": [{"given": "Ankitaben", "family": "Mungalpara", "sequence": "first", "affiliation": [], "role": [{"vocabulary": "crossref", "role": "author"}]}], "member": "12709", "contributor": [{"sequence": "additional", "affiliation": [], "role": [null]}], "deposited": {"date-parts": [[2026, 4, 7]], "date-time": "2026-04-07T05:15:59Z", "timestamp": 1775538959000}, "score": 23.31637, "degree": ["Master of Science (MS)"], "resource": {"primary": {"URL": "https://repository.lib.umassd.edu/esploro/outputs/graduate/9914504462201301"}}, "subtitle": ["instruction tuning with region of interest attention : a thesis in Data Science"], "issued": {"date-parts": [[null]]}, "references-count": 0, "URL": "https://doi.org/10.62791/20485"}], "items-per-page": 6, "query": {"start-index": 0, "search-terms": null}}}