[{"user_id":"62841","doi":"10.17619/UNIPB/1-2646","editor":[{"last_name":"Graessler","first_name":"Iris","full_name":"Graessler, Iris"}],"_id":"66837","language":[{"iso":"eng"}],"publisher":"Universitätsbibliothek","date_updated":"2026-08-24T14:41:41Z","has_accepted_license":"1","title":"Performance evaluation of large language models in extracting requirements sets from technical documents","status":"public","year":"2026","author":[{"full_name":"Gräßler, Iris","first_name":"Iris","orcid":"0000-0001-5765-971X","last_name":"Gräßler","id":"47565"},{"id":"62841","full_name":"Pfeifer, Jan Niklas","first_name":"Jan Niklas","last_name":"Pfeifer"}],"conference":{"end_date":"26.03.2026","start_date":"24.03.2026","name":"International Symposium on Hybrid Intelligence in Product and Production Engineering 1. 2026 Paderborn","location":"Paderborn"},"type":"conference","keyword":["Requirements Engineering","Artificial Intelligence","Large Language Models","Generative AI"],"date_created":"2026-08-24T14:31:30Z","quality_controlled":"1","abstract":[{"text":"Manufacturers operate under dynamic regulatory conditions, leading to frequently changing requirements that must be managed with high effort. AI-based requirements extraction using Large Language Models (LLMs) has the potential to support engineers in this task by analyzing large documents, for example. Existing evaluation approaches in AI-supported requirements engineering mainly assess individual requirements and their consistency with source documents, while quality criteria of requirements sets are largely neglected. This paper presents a five-step approach to evaluate the performance of current LLMs in generating requirements sets: a systematic literature review to identify quality criteria of requirements sets (1), exemplary generation of requirements sets from the Machinery Directive 2006/42/EC using LLMs (2), examination (3), evaluation (4) and comparison (5) of the generated requirements sets. The results support engineers in selecting LLMs for creating requirement sets and in evaluating future LLMs.","lang":"eng"}],"publication":"1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University","citation":{"mla":"Gräßler, Iris, and Jan Niklas Pfeifer. “Performance Evaluation of Large Language Models in Extracting Requirements Sets from Technical Documents.” <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, edited by Iris Graessler, Universitätsbibliothek, 2026, doi:<a href=\"https://doi.org/10.17619/UNIPB/1-2646\">10.17619/UNIPB/1-2646</a>.","ama":"Gräßler I, Pfeifer JN. Performance evaluation of large language models in extracting requirements sets from technical documents. In: Graessler I, ed. <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>. Universitätsbibliothek; 2026. doi:<a href=\"https://doi.org/10.17619/UNIPB/1-2646\">10.17619/UNIPB/1-2646</a>","bibtex":"@inproceedings{Gräßler_Pfeifer_2026, title={Performance evaluation of large language models in extracting requirements sets from technical documents}, DOI={<a href=\"https://doi.org/10.17619/UNIPB/1-2646\">10.17619/UNIPB/1-2646</a>}, booktitle={1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University}, publisher={Universitätsbibliothek}, author={Gräßler, Iris and Pfeifer, Jan Niklas}, editor={Graessler, Iris}, year={2026} }","apa":"Gräßler, I., &#38; Pfeifer, J. N. (2026). Performance evaluation of large language models in extracting requirements sets from technical documents. In I. Graessler (Ed.), <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>. Universitätsbibliothek. <a href=\"https://doi.org/10.17619/UNIPB/1-2646\">https://doi.org/10.17619/UNIPB/1-2646</a>","ieee":"I. Gräßler and J. N. Pfeifer, “Performance evaluation of large language models in extracting requirements sets from technical documents,” in <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, Paderborn, 2026, doi: <a href=\"https://doi.org/10.17619/UNIPB/1-2646\">10.17619/UNIPB/1-2646</a>.","chicago":"Gräßler, Iris, and Jan Niklas Pfeifer. “Performance Evaluation of Large Language Models in Extracting Requirements Sets from Technical Documents.” In <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, edited by Iris Graessler. Universitätsbibliothek, 2026. <a href=\"https://doi.org/10.17619/UNIPB/1-2646\">https://doi.org/10.17619/UNIPB/1-2646</a>.","short":"I. Gräßler, J.N. Pfeifer, in: I. Graessler (Ed.), 1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University, Universitätsbibliothek, 2026."}},{"place":"Paderborn","quality_controlled":"1","citation":{"apa":"Gräßler, I., Mansheim, J., &#38; Pfeifer, J. N. (2026). Harnessing tacit Knowledge from Sustainable Product Engineering through Artificial Intelligence. In I. Graessler (Ed.), <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i> (Vol. 1, pp. 229–238). Universitätsbibliothek. <a href=\"https://doi.org/10.17619/UNIPB/1-2639\">https://doi.org/10.17619/UNIPB/1-2639</a>","ieee":"I. Gräßler, J. Mansheim, and J. N. Pfeifer, “Harnessing tacit Knowledge from Sustainable Product Engineering through Artificial Intelligence,” in <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, Paderborn, 2026, vol. 1, pp. 229–238, doi: <a href=\"https://doi.org/10.17619/UNIPB/1-2639\">10.17619/UNIPB/1-2639</a>.","short":"I. Gräßler, J. Mansheim, J.N. Pfeifer, in: I. Graessler (Ed.), 1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University, Universitätsbibliothek, Paderborn, 2026, pp. 229–238.","chicago":"Gräßler, Iris, Johanna Mansheim, and Jan Niklas Pfeifer. “Harnessing Tacit Knowledge from Sustainable Product Engineering through Artificial Intelligence.” In <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, edited by Iris Graessler, 1:229–38. International Symposium on Hybrid Intelligence in Product and Production Engineering. Paderborn: Universitätsbibliothek, 2026. <a href=\"https://doi.org/10.17619/UNIPB/1-2639\">https://doi.org/10.17619/UNIPB/1-2639</a>.","mla":"Gräßler, Iris, et al. “Harnessing Tacit Knowledge from Sustainable Product Engineering through Artificial Intelligence.” <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, edited by Iris Graessler, vol. 1, Universitätsbibliothek, 2026, pp. 229–38, doi:<a href=\"https://doi.org/10.17619/UNIPB/1-2639\">10.17619/UNIPB/1-2639</a>.","ama":"Gräßler I, Mansheim J, Pfeifer JN. Harnessing tacit Knowledge from Sustainable Product Engineering through Artificial Intelligence. In: Graessler I, ed. <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>. Vol 1. International Symposium on Hybrid Intelligence in Product and Production Engineering. Universitätsbibliothek; 2026:229-238. doi:<a href=\"https://doi.org/10.17619/UNIPB/1-2639\">10.17619/UNIPB/1-2639</a>","bibtex":"@inproceedings{Gräßler_Mansheim_Pfeifer_2026, place={Paderborn}, series={International Symposium on Hybrid Intelligence in Product and Production Engineering}, title={Harnessing tacit Knowledge from Sustainable Product Engineering through Artificial Intelligence}, volume={1}, DOI={<a href=\"https://doi.org/10.17619/UNIPB/1-2639\">10.17619/UNIPB/1-2639</a>}, booktitle={1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University}, publisher={Universitätsbibliothek}, author={Gräßler, Iris and Mansheim, Johanna and Pfeifer, Jan Niklas}, editor={Graessler, Iris}, year={2026}, pages={229–238}, collection={International Symposium on Hybrid Intelligence in Product and Production Engineering} }"},"volume":1,"editor":[{"last_name":"Graessler","first_name":"Iris","full_name":"Graessler, Iris"}],"user_id":"98210","_id":"66547","publisher":"Universitätsbibliothek","page":"229-238","conference":{"end_date":"2026-03-26","start_date":"2026-03-24","name":"International Symposium on Hybrid Intelligence in Product and Production Engineering 1. 2026 Paderborn","location":"Paderborn"},"status":"public","department":[{"_id":"152"}],"keyword":["Tacit knowledge","Sustainability","Product Engineering","Artificial Intelligence"],"type":"conference","date_created":"2026-07-20T13:16:49Z","abstract":[{"lang":"eng","text":"Tacit knowledge is particularly valuable in product engineering. Experiences reside in minds of employees and are not systematically documented but could have a significant influence on sustainable product engineering. Existing approaches do not offer sufficient support for harnessing technical tacit knowledge regarding sustainable product engineering. In this paper a three-step method is presented: The method contains knowledge acquisition, requirements extraction und requirements validation for quality assurance. Acquisition is performed by support artifacts like interview guidelines. Extraction is supported by Artificial Intelligence, converting statements into requirements. These requirements are validated using a specific questionnaire. The developed approach is applied in a funded project of the European Union (EU) with automotive industry partners. The method supports product engineers to collect tacit knowledge from experienced employees, transform unsystematic knowledge into standardized technical requirements and validate them. Application of this method enables creation of validated requirements that can be applied throughout the company and are not exclusively dependent on a single employee."}],"publication":"1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University","doi":"10.17619/UNIPB/1-2639","series_title":"International Symposium on Hybrid Intelligence in Product and Production Engineering","language":[{"iso":"eng"}],"intvolume":"         1","publication_status":"published","date_updated":"2026-08-31T08:00:06Z","author":[{"full_name":"Gräßler, Iris","last_name":"Gräßler","orcid":"0000-0001-5765-971X","first_name":"Iris","id":"47565"},{"first_name":"Johanna","last_name":"Mansheim","full_name":"Mansheim, Johanna","id":"98210"},{"first_name":"Jan Niklas","last_name":"Pfeifer","full_name":"Pfeifer, Jan Niklas","id":"62841"}],"title":"Harnessing tacit Knowledge from Sustainable Product Engineering through Artificial Intelligence","year":"2026"},{"date_created":"2024-09-11T08:07:26Z","department":[{"_id":"152"}],"keyword":["Systems Engineering roles","properties","features","emergent phenomena","automotive"],"type":"conference","issue":"1","publication":"INCOSE International Symposium","abstract":[{"text":"<jats:title>Abstract</jats:title><jats:p>Engineering projects for complex technical systems such as automobiles demand extensive requirement specifications and corresponding hierarchy levels in system architectures. Especially when considering emergent phenomena, such as total weight or aerodynamics, a closely networked collaboration of discipline‐specific and cross‐disciplinary roles is required. Further, in large organizations with a group structure, resulting functional and non‐functional contents need to be managed by distinct Systems Engineering roles. For example, the role “property manager” takes care of achieving overarching product properties, such as weight or aerodynamics, which cannot be directly fixed, but result from many specifications. This paper proposes new Systems Engineering roles and their application in a German Original Equipment Manufacturer (OEM). For validation, the roles have already been applied in everyday engineering projects at the OEM. The concept proved to be indispensable and transferable to other Systems Engineering projects.</jats:p>","lang":"eng"}],"language":[{"iso":"eng"}],"doi":"10.1002/iis2.13164","author":[{"full_name":"Gräßler, Iris","orcid":"0000-0001-5765-971X","last_name":"Gräßler","first_name":"Iris","id":"47565"},{"id":"62841","full_name":"Pfeifer, Jan Niklas","first_name":"Jan Niklas","last_name":"Pfeifer"},{"first_name":"Florian","last_name":"Hintz","full_name":"Hintz, Florian"},{"full_name":"Meyrl, Nicolas","first_name":"Nicolas","last_name":"Meyrl"}],"publication_identifier":{"issn":["2334-5837","2334-5837"]},"title":"Systems Engineering roles to handle emergent properties and behaviors in complex technical systems","year":"2024","intvolume":"        34","date_updated":"2024-09-11T08:22:55Z","publication_status":"published","citation":{"short":"I. Gräßler, J.N. Pfeifer, F. Hintz, N. Meyrl, in: INCOSE International Symposium, Wiley, 2024, pp. 584–598.","chicago":"Gräßler, Iris, Jan Niklas Pfeifer, Florian Hintz, and Nicolas Meyrl. “Systems Engineering Roles to Handle Emergent Properties and Behaviors in Complex Technical Systems.” In <i>INCOSE International Symposium</i>, 34:584–98. Wiley, 2024. <a href=\"https://doi.org/10.1002/iis2.13164\">https://doi.org/10.1002/iis2.13164</a>.","ieee":"I. Gräßler, J. N. Pfeifer, F. Hintz, and N. Meyrl, “Systems Engineering roles to handle emergent properties and behaviors in complex technical systems,” in <i>INCOSE International Symposium</i>, Dublin, Ireland, 2024, vol. 34, no. 1, pp. 584–598, doi: <a href=\"https://doi.org/10.1002/iis2.13164\">10.1002/iis2.13164</a>.","apa":"Gräßler, I., Pfeifer, J. N., Hintz, F., &#38; Meyrl, N. (2024). Systems Engineering roles to handle emergent properties and behaviors in complex technical systems. <i>INCOSE International Symposium</i>, <i>34</i>(1), 584–598. <a href=\"https://doi.org/10.1002/iis2.13164\">https://doi.org/10.1002/iis2.13164</a>","bibtex":"@inproceedings{Gräßler_Pfeifer_Hintz_Meyrl_2024, title={Systems Engineering roles to handle emergent properties and behaviors in complex technical systems}, volume={34}, DOI={<a href=\"https://doi.org/10.1002/iis2.13164\">10.1002/iis2.13164</a>}, number={1}, booktitle={INCOSE International Symposium}, publisher={Wiley}, author={Gräßler, Iris and Pfeifer, Jan Niklas and Hintz, Florian and Meyrl, Nicolas}, year={2024}, pages={584–598} }","ama":"Gräßler I, Pfeifer JN, Hintz F, Meyrl N. Systems Engineering roles to handle emergent properties and behaviors in complex technical systems. In: <i>INCOSE International Symposium</i>. Vol 34. Wiley; 2024:584-598. doi:<a href=\"https://doi.org/10.1002/iis2.13164\">10.1002/iis2.13164</a>","mla":"Gräßler, Iris, et al. “Systems Engineering Roles to Handle Emergent Properties and Behaviors in Complex Technical Systems.” <i>INCOSE International Symposium</i>, vol. 34, no. 1, Wiley, 2024, pp. 584–98, doi:<a href=\"https://doi.org/10.1002/iis2.13164\">10.1002/iis2.13164</a>."},"quality_controlled":"1","_id":"56108","publisher":"Wiley","page":"584-598","volume":34,"user_id":"62841","conference":{"end_date":"2024-07-06","location":"Dublin, Ireland","start_date":"2024-07-02","name":"34th Annual INCOSE International Symposium"},"status":"public"}]