[{"citation":{"mla":"Gräßler, Iris, and Jan Niklas Pfeifer. “Performance Evaluation of Large Language Models in Extracting Requirements Sets from Technical Documents.” <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, edited by Iris Graessler, Universitätsbibliothek, 2026, doi:<a href=\"https://doi.org/10.17619/UNIPB/1-2646\">10.17619/UNIPB/1-2646</a>.","bibtex":"@inproceedings{Gräßler_Pfeifer_2026, title={Performance evaluation of large language models in extracting requirements sets from technical documents}, DOI={<a href=\"https://doi.org/10.17619/UNIPB/1-2646\">10.17619/UNIPB/1-2646</a>}, booktitle={1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University}, publisher={Universitätsbibliothek}, author={Gräßler, Iris and Pfeifer, Jan Niklas}, editor={Graessler, Iris}, year={2026} }","ama":"Gräßler I, Pfeifer JN. Performance evaluation of large language models in extracting requirements sets from technical documents. In: Graessler I, ed. <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>. Universitätsbibliothek; 2026. doi:<a href=\"https://doi.org/10.17619/UNIPB/1-2646\">10.17619/UNIPB/1-2646</a>","ieee":"I. Gräßler and J. N. Pfeifer, “Performance evaluation of large language models in extracting requirements sets from technical documents,” in <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, Paderborn, 2026, doi: <a href=\"https://doi.org/10.17619/UNIPB/1-2646\">10.17619/UNIPB/1-2646</a>.","apa":"Gräßler, I., &#38; Pfeifer, J. N. (2026). Performance evaluation of large language models in extracting requirements sets from technical documents. In I. Graessler (Ed.), <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>. Universitätsbibliothek. <a href=\"https://doi.org/10.17619/UNIPB/1-2646\">https://doi.org/10.17619/UNIPB/1-2646</a>","chicago":"Gräßler, Iris, and Jan Niklas Pfeifer. “Performance Evaluation of Large Language Models in Extracting Requirements Sets from Technical Documents.” In <i>1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University</i>, edited by Iris Graessler. Universitätsbibliothek, 2026. <a href=\"https://doi.org/10.17619/UNIPB/1-2646\">https://doi.org/10.17619/UNIPB/1-2646</a>.","short":"I. Gräßler, J.N. Pfeifer, in: I. Graessler (Ed.), 1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University, Universitätsbibliothek, 2026."},"publication":"1st International Symposium: March 24 – 26, 2026, Heinz Nixdorf Institute, Paderborn University","quality_controlled":"1","abstract":[{"text":"Manufacturers operate under dynamic regulatory conditions, leading to frequently changing requirements that must be managed with high effort. AI-based requirements extraction using Large Language Models (LLMs) has the potential to support engineers in this task by analyzing large documents, for example. Existing evaluation approaches in AI-supported requirements engineering mainly assess individual requirements and their consistency with source documents, while quality criteria of requirements sets are largely neglected. This paper presents a five-step approach to evaluate the performance of current LLMs in generating requirements sets: a systematic literature review to identify quality criteria of requirements sets (1), exemplary generation of requirements sets from the Machinery Directive 2006/42/EC using LLMs (2), examination (3), evaluation (4) and comparison (5) of the generated requirements sets. The results support engineers in selecting LLMs for creating requirement sets and in evaluating future LLMs.","lang":"eng"}],"date_created":"2026-08-24T14:31:30Z","keyword":["Requirements Engineering","Artificial Intelligence","Large Language Models","Generative AI"],"type":"conference","author":[{"full_name":"Gräßler, Iris","first_name":"Iris","orcid":"0000-0001-5765-971X","last_name":"Gräßler","id":"47565"},{"full_name":"Pfeifer, Jan Niklas","last_name":"Pfeifer","first_name":"Jan Niklas","id":"62841"}],"conference":{"location":"Paderborn","start_date":"24.03.2026","name":"International Symposium on Hybrid Intelligence in Product and Production Engineering 1. 2026 Paderborn","end_date":"26.03.2026"},"year":"2026","title":"Performance evaluation of large language models in extracting requirements sets from technical documents","status":"public","has_accepted_license":"1","date_updated":"2026-08-24T14:41:41Z","_id":"66837","publisher":"Universitätsbibliothek","language":[{"iso":"eng"}],"editor":[{"full_name":"Graessler, Iris","last_name":"Graessler","first_name":"Iris"}],"user_id":"62841","doi":"10.17619/UNIPB/1-2646"},{"year":"2019","title":"Requirements Engineering in OTF-Computing","author":[{"last_name":"Bäumer","first_name":"Frederik Simon","full_name":"Bäumer, Frederik Simon","id":"38837"},{"id":"42496","orcid":"0000-0002-8180-5606","last_name":"Geierhos","first_name":"Michaela","full_name":"Geierhos, Michaela"}],"date_updated":"2022-01-06T07:03:53Z","publication_status":"published","main_file_link":[{"open_access":"1","url":"https://encyclopedia.pub/131"}],"language":[{"iso":"eng"}],"publication":"encyclopedia.pub","date_created":"2019-03-05T08:54:37Z","keyword":["OTF Computing","Natural Language Processing","Requirements Engineering"],"type":"encyclopedia_article","department":[{"_id":"36"},{"_id":"1"},{"_id":"579"}],"status":"public","_id":"8312","publisher":"MDPI","user_id":"42496","citation":{"mla":"Bäumer, Frederik Simon, and Michaela Geierhos. “Requirements Engineering in OTF-Computing.” <i>Encyclopedia.Pub</i>, MDPI, 2019.","ama":"Bäumer FS, Geierhos M. Requirements Engineering in OTF-Computing. In: <i>Encyclopedia.Pub</i>. Basel, Switzerland: MDPI; 2019.","bibtex":"@inbook{Bäumer_Geierhos_2019, place={Basel, Switzerland}, title={Requirements Engineering in OTF-Computing}, booktitle={encyclopedia.pub}, publisher={MDPI}, author={Bäumer, Frederik Simon and Geierhos, Michaela}, year={2019} }","apa":"Bäumer, F. S., &#38; Geierhos, M. (2019). Requirements Engineering in OTF-Computing. In <i>encyclopedia.pub</i>. Basel, Switzerland: MDPI.","ieee":"F. S. Bäumer and M. Geierhos, “Requirements Engineering in OTF-Computing,” in <i>encyclopedia.pub</i>, Basel, Switzerland: MDPI, 2019.","chicago":"Bäumer, Frederik Simon, and Michaela Geierhos. “Requirements Engineering in OTF-Computing.” In <i>Encyclopedia.Pub</i>. Basel, Switzerland: MDPI, 2019.","short":"F.S. Bäumer, M. Geierhos, in: Encyclopedia.Pub, MDPI, Basel, Switzerland, 2019."},"quality_controlled":"1","project":[{"_id":"1","name":"SFB 901"},{"name":"SFB 901 - Project Area B","_id":"3"},{"_id":"9","name":"SFB 901 - Subproject B1"}],"place":"Basel, Switzerland","oa":"1"},{"date_created":"2018-04-13T09:44:09Z","keyword":["Digitalisierungspotenziale","BPI","Digitale Transformation","Information Flow-Modellierung","Patterns","Requirements Engineering"],"type":"conference","department":[{"_id":"66"}],"publication":"Joint Proceedings of the Workshops at Modellierung 2018 co-located with Modellierung 2018, Braunschweig, Germany, February 21, 2018.","abstract":[{"text":"Ein wichtiges Element der Digitalen Transformation ist die Digitalisierung der Prozesse in Unternehmen. Eine Herausforderung besteht hierbei in der systematischen Erkennung von Digitalisierungspotenzialen in Prozessen. Bestehende Ansätze benötigen Experten, welche Potenziale über ihre Erfahrung oder zeitaufwendig mithilfe von Musterkatalogen identifizieren.\r\nIn diesem Artikel werden verschiedene Digitalisierungspotenziale klassifiziert und Muster für ein zukünftiges musterbasiertes Analyseverfahren zur automatisierten Identifikation von Digitalisierungspotenzialen in BPMN-Diagrammen beschrieben. Im Vergleich zu bestehenden Ansätzen erlaubt es Experten die Identifizierung von Digitalisierungspotenzialen effizienter und effektiver durchzuführen.","lang":"ger"},{"text":"An important element of digital transformation is the digitalization of processes within enterprises.\r\nA major challenge is the systematic identification of digitalization potentials in business processes.\r\nExisting approaches require experts who identify said potentials by using the time-consuming method of pattern catalogs or by relying on their professional experiences. \r\nIn this paper we classify potentials of digitalization and corresponding patterns for a future pattern-based analysis procedure. This shall allow for the automated identification of digitalization potentials in BPMN diagrams. In comparison to existing approaches, our proposed method could support a more efficient and effective identification of digitalization potentials by experts.","lang":"eng"}],"main_file_link":[{"url":"http://ceur-ws.org/Vol-2060/rebpm2.pdf","open_access":"1"}],"series_title":"CEUR Workshop Proceedings","language":[{"iso":"ger"}],"year":"2018","title":"Digitalisierungspotenziale in Geschäftsprozessen effizient und effektiv erkennen (Effective and Efficient Identification of Digitalization Potentials in Business Processes)","author":[{"id":"5281","full_name":"Rittmeier, Florian","last_name":"Rittmeier","first_name":"Florian"},{"id":"107","full_name":"Engels, Gregor","last_name":"Engels","first_name":"Gregor"},{"last_name":"Teetz","first_name":"Alexander","full_name":"Teetz, Alexander","id":"5319"}],"publication_status":"published","date_updated":"2022-01-06T06:55:49Z","intvolume":"      2060","oa":"1","citation":{"mla":"Rittmeier, Florian, et al. “Digitalisierungspotenziale in Geschäftsprozessen effizient und effektiv erkennen (Effective and Efficient Identification of Digitalization Potentials in Business Processes).” <i>Joint Proceedings of the Workshops at Modellierung 2018 co-located with Modellierung 2018, Braunschweig, Germany, February 21, 2018.</i>, vol. 2060, CEUR-WS.org, 2018, pp. 215--221.","ama":"Rittmeier F, Engels G, Teetz A. Digitalisierungspotenziale in Geschäftsprozessen effizient und effektiv erkennen (Effective and Efficient Identification of Digitalization Potentials in Business Processes). In: <i>Joint Proceedings of the Workshops at Modellierung 2018 co-located with Modellierung 2018, Braunschweig, Germany, February 21, 2018.</i> Vol 2060. CEUR Workshop Proceedings. CEUR-WS.org; 2018:215--221.","bibtex":"@inproceedings{Rittmeier_Engels_Teetz_2018, series={CEUR Workshop Proceedings}, title={Digitalisierungspotenziale in Geschäftsprozessen effizient und effektiv erkennen (Effective and Efficient Identification of Digitalization Potentials in Business Processes)}, volume={2060}, booktitle={Joint Proceedings of the Workshops at Modellierung 2018 co-located with Modellierung 2018, Braunschweig, Germany, February 21, 2018.}, publisher={CEUR-WS.org}, author={Rittmeier, Florian and Engels, Gregor and Teetz, Alexander}, year={2018}, pages={215--221}, collection={CEUR Workshop Proceedings} }","apa":"Rittmeier, F., Engels, G., &#38; Teetz, A. (2018). Digitalisierungspotenziale in Geschäftsprozessen effizient und effektiv erkennen (Effective and Efficient Identification of Digitalization Potentials in Business Processes). In <i>Joint Proceedings of the Workshops at Modellierung 2018 co-located with Modellierung 2018, Braunschweig, Germany, February 21, 2018.</i> (Vol. 2060, pp. 215--221). CEUR-WS.org.","ieee":"F. Rittmeier, G. Engels, and A. Teetz, “Digitalisierungspotenziale in Geschäftsprozessen effizient und effektiv erkennen (Effective and Efficient Identification of Digitalization Potentials in Business Processes),” in <i>Joint Proceedings of the Workshops at Modellierung 2018 co-located with Modellierung 2018, Braunschweig, Germany, February 21, 2018.</i>, 2018, vol. 2060, pp. 215--221.","short":"F. Rittmeier, G. Engels, A. Teetz, in: Joint Proceedings of the Workshops at Modellierung 2018 co-located with Modellierung 2018, Braunschweig, Germany, February 21, 2018., CEUR-WS.org, 2018, pp. 215--221.","chicago":"Rittmeier, Florian, Gregor Engels, and Alexander Teetz. “Digitalisierungspotenziale in Geschäftsprozessen effizient und effektiv erkennen (Effective and Efficient Identification of Digitalization Potentials in Business Processes).” In <i>Joint Proceedings of the Workshops at Modellierung 2018 co-located with Modellierung 2018, Braunschweig, Germany, February 21, 2018.</i>, 2060:215--221. CEUR Workshop Proceedings. CEUR-WS.org, 2018."},"page":"215--221","publisher":"CEUR-WS.org","_id":"2332","user_id":"5281","volume":2060,"status":"public"},{"author":[{"id":"58843","full_name":"van Rooijen, Lorijn","last_name":"van Rooijen","first_name":"Lorijn"},{"id":"38837","full_name":"Bäumer, Frederik Simon","last_name":"Bäumer","first_name":"Frederik Simon"},{"first_name":"Marie Christin","last_name":"Platenius","full_name":"Platenius, Marie Christin"},{"id":"42496","first_name":"Michaela","last_name":"Geierhos","orcid":"0000-0002-8180-5606","full_name":"Geierhos, Michaela"},{"full_name":"Hamann, Heiko","first_name":"Heiko","last_name":"Hamann"},{"id":"107","first_name":"Gregor","last_name":"Engels","full_name":"Engels, Gregor"}],"publication_identifier":{"eisbn":["978-1-5386-3488-2 "],"isbn":["978-1-5386-3489-9"]},"year":"2017","title":"From User Demand to Software Service: Using Machine Learning to Automate the Requirements Specification Process","publication_status":"published","date_updated":"2022-01-06T07:04:18Z","language":[{"iso":"eng"}],"doi":"10.1109/REW.2017.26","publication":"2017 IEEE 25th International Requirements Engineering Conference Workshops (REW)","abstract":[{"lang":"eng","text":"Bridging the gap between informal, imprecise, and vague user requirements descriptions and precise formalized specifications is the main task of requirements engineering. Techniques such as interviews or story telling are used when requirements engineers try to identify a user's needs. The requirements specification process is typically done in a dialogue between users, domain experts, and requirements engineers. In our research, we aim at automating the specification of requirements. The idea is to distinguish between untrained users and trained users, and to exploit domain knowledge learned from previous runs of our system. We let untrained users provide unstructured natural language descriptions, while we allow trained users to provide examples of behavioral descriptions. In both cases, our goal is to synthesize formal requirements models similar to statecharts. From requirements specification processes with trained users, behavioral ontologies are learned which are later used to support the requirements specification process for untrained users. Our research method is original in combining natural language processing and search-based techniques for the synthesis of requirements specifications. Our work is embedded in a larger project that aims at automating the whole software development and deployment process in envisioned future software service markets."}],"date_created":"2017-10-17T12:41:10Z","file":[{"creator":"ups","date_created":"2018-11-02T14:50:35Z","file_name":"08054881.pdf","file_size":433613,"access_level":"closed","relation":"main_file","date_updated":"2018-11-02T14:50:35Z","file_id":"5285","content_type":"application/pdf","success":1}],"department":[{"_id":"36"},{"_id":"1"},{"_id":"579"},{"_id":"34"},{"_id":"7"},{"_id":"66"},{"_id":"238"},{"_id":"63"}],"type":"conference","keyword":["Software","Unified modeling language","Requirements engineering","Ontologies","Search problems","Natural languages"],"conference":{"end_date":"2017-09-08","start_date":"2017-09-04","name":"2017 IEEE 25th International Requirements Engineering Conference Workshops (REW)","location":"Lisbon, Portugal"},"status":"public","has_accepted_license":"1","_id":"97","publisher":"IEEE","page":"379-385","user_id":"57458","ddc":["000"],"citation":{"chicago":"Rooijen, Lorijn van, Frederik Simon Bäumer, Marie Christin Platenius, Michaela Geierhos, Heiko Hamann, and Gregor Engels. “From User Demand to Software Service: Using Machine Learning to Automate the Requirements Specification Process.” In <i>2017 IEEE 25th International Requirements Engineering Conference Workshops (REW)</i>, 379–85. Piscataway, NJ, USA: IEEE, 2017. <a href=\"https://doi.org/10.1109/REW.2017.26\">https://doi.org/10.1109/REW.2017.26</a>.","short":"L. van Rooijen, F.S. Bäumer, M.C. Platenius, M. Geierhos, H. Hamann, G. Engels, in: 2017 IEEE 25th International Requirements Engineering Conference Workshops (REW), IEEE, Piscataway, NJ, USA, 2017, pp. 379–385.","ieee":"L. van Rooijen, F. S. Bäumer, M. C. Platenius, M. Geierhos, H. Hamann, and G. Engels, “From User Demand to Software Service: Using Machine Learning to Automate the Requirements Specification Process,” in <i>2017 IEEE 25th International Requirements Engineering Conference Workshops (REW)</i>, Lisbon, Portugal, 2017, pp. 379–385.","apa":"van Rooijen, L., Bäumer, F. S., Platenius, M. C., Geierhos, M., Hamann, H., &#38; Engels, G. (2017). From User Demand to Software Service: Using Machine Learning to Automate the Requirements Specification Process. In <i>2017 IEEE 25th International Requirements Engineering Conference Workshops (REW)</i> (pp. 379–385). Piscataway, NJ, USA: IEEE. <a href=\"https://doi.org/10.1109/REW.2017.26\">https://doi.org/10.1109/REW.2017.26</a>","bibtex":"@inproceedings{van Rooijen_Bäumer_Platenius_Geierhos_Hamann_Engels_2017, place={Piscataway, NJ, USA}, title={From User Demand to Software Service: Using Machine Learning to Automate the Requirements Specification Process}, DOI={<a href=\"https://doi.org/10.1109/REW.2017.26\">10.1109/REW.2017.26</a>}, booktitle={2017 IEEE 25th International Requirements Engineering Conference Workshops (REW)}, publisher={IEEE}, author={van Rooijen, Lorijn and Bäumer, Frederik Simon and Platenius, Marie Christin and Geierhos, Michaela and Hamann, Heiko and Engels, Gregor}, year={2017}, pages={379–385} }","ama":"van Rooijen L, Bäumer FS, Platenius MC, Geierhos M, Hamann H, Engels G. From User Demand to Software Service: Using Machine Learning to Automate the Requirements Specification Process. In: <i>2017 IEEE 25th International Requirements Engineering Conference Workshops (REW)</i>. Piscataway, NJ, USA: IEEE; 2017:379-385. doi:<a href=\"https://doi.org/10.1109/REW.2017.26\">10.1109/REW.2017.26</a>","mla":"van Rooijen, Lorijn, et al. “From User Demand to Software Service: Using Machine Learning to Automate the Requirements Specification Process.” <i>2017 IEEE 25th International Requirements Engineering Conference Workshops (REW)</i>, IEEE, 2017, pp. 379–85, doi:<a href=\"https://doi.org/10.1109/REW.2017.26\">10.1109/REW.2017.26</a>."},"file_date_updated":"2018-11-02T14:50:35Z","project":[{"name":"SFB 901","_id":"1"},{"_id":"9","name":"SFB 901 - Subprojekt B1"},{"_id":"3","name":"SFB 901 - Project Area B"}],"quality_controlled":"1","place":"Piscataway, NJ, USA"},{"publication_status":"published","date_updated":"2022-12-30T22:12:13Z","has_accepted_license":"1","year":"2015","status":"public","title":"Early Prediction of Security Properties for Mechatronic Systems","author":[{"id":"36399","full_name":"Schwichtenberg, Bahar","last_name":"Schwichtenberg","first_name":"Bahar"}],"user_id":"36399","ddc":["000"],"language":[{"iso":"eng"}],"_id":"33312","extern":"1","abstract":[{"lang":"eng","text":"Mechatronic systems are used more than ever in human life. They can be found in a very wide range of domain contexts, from household appliances, and cars, to medical equipment. Mechatronic systems, as a kind of embedded systems, are the tight integration of mechanical and electrical engineering, which embed software systems. Information security of mechatronic systems has not received much attention yet. However, wherever data exists, cyber attacks threaten mechatronic systems.\r\n\r\nThe thesis focuses on the early design stages of the development of mechatronic systems. Model sequence diagrams (MSDs) are used to model requirements with real-time and safety properties. In this thesis, MSDs are extended such that security properties for example authenticity and privacy can be modeled and analyzed automatically."}],"file_date_updated":"2022-12-30T22:10:51Z","citation":{"ieee":"B. Schwichtenberg, <i>Early Prediction of Security Properties for Mechatronic Systems</i>. 2015.","apa":"Schwichtenberg, B. (2015). <i>Early Prediction of Security Properties for Mechatronic Systems</i>.","short":"B. Schwichtenberg, Early Prediction of Security Properties for Mechatronic Systems, 2015.","chicago":"Schwichtenberg, Bahar. <i>Early Prediction of Security Properties for Mechatronic Systems</i>, 2015.","mla":"Schwichtenberg, Bahar. <i>Early Prediction of Security Properties for Mechatronic Systems</i>. 2015.","bibtex":"@book{Schwichtenberg_2015, title={Early Prediction of Security Properties for Mechatronic Systems}, author={Schwichtenberg, Bahar}, year={2015} }","ama":"Schwichtenberg B. <i>Early Prediction of Security Properties for Mechatronic Systems</i>.; 2015."},"keyword":["Software Architecture","Requirements Engineering","Embedded Systems"],"type":"mastersthesis","file":[{"date_updated":"2022-12-30T22:10:51Z","relation":"main_file","file_size":11423528,"access_level":"closed","file_name":"Bahar_Jazayeri_Masterarbeit.pdf","content_type":"application/pdf","success":1,"file_id":"35068","creator":"bahareh","date_created":"2022-12-30T22:10:51Z"}],"date_created":"2022-09-09T11:42:25Z"}]