% !TEX program = lualatex
% !BIB program = biber
%                                                __
%  __                                     __    /\ \
% /\_\    ___      __          __   _ __ /\_\   \_\ \
% \/\ \ /' _ `\  /'_ `\      /'_ `\/\`'__\/\ \  /'_` \
%  \ \ \/\ \/\ \/\ \L\ \  __/\ \L\ \ \ \/ \ \ \/\ \L\ \
%   \ \_\ \_\ \_\ \____ \/\_\ \____ \ \_\  \ \_\ \___,_\
%    \/_/\/_/\/_/\/___L\ \/_/\/___L\ \/_/   \/_/\/__,_ /
%                  /\____/     /\____/
%                  \_/__/      \_/__/
%
%                        journal ing.grid
%
%
% Template for authors
%
% Structure of this template:
% .
% ├── acknowledgements.tex <-- acknowledgements, funding information
% ├── authors.tex         <-- authors and affiliations
% ├── figures             <-- images and pictures
% │   └── image-a.pdf         example image
% ├── fonts               <-- needed fonts
% ├── inggrid.cls         <-- the documentclass
% ├── main.tex            <-- this file, content
% └── references.bib      <-- the bibliography
%
%
\documentclass{inggrid}
%
%
% The documentclass already loads a number of packages, amongst others:
% - amsmath,amssym
% - xcolor, graphicx
% - babel, csquotes
% - biblatex
% - fontspec, microtype
% - booktabs, longtable
% - listings
%
% -----------------------------------------------
% Custom packages


\usepackage{booktabs}
\usepackage{subcaption}
\usepackage{pifont}
\usepackage{fdsymbol}
\usepackage{wasysym}

% Only during rewrite
% \usepackage{todonotes}
% \usepackage{marginnote}
% \let\marginpar\marginnote

% \newcommand{\tododaniel}[1]{\todo[backgroundcolor=green!30!white]{DH: #1}}
% \newcommand{\todoflorian}[1]{\todo[backgroundcolor=blue!30!white]{FS: #1}}
% \newcommand{\todoalex}[1]{\todo[backgroundcolor=orange!30!white]{AS: #1}}
% \newcommand{\todoinlinedaniel}[1]{\todo[backgroundcolor=green!30!white,inline,caption={}]{DH: #1}}
% \newcommand{\todoinlineflorian}[1]{\todo[backgroundcolor=blue!30!white,inline,caption={}]{FS: #1}}
% \newcommand{\todoinlinealex}[1]{\todo[backgroundcolor=orange!30!white,inline,caption={}]{AS: #1}}

\newcommand{\checkdh}[0]{\ding{52}}
\newcommand{\crossdh}[0]{\ding{55}}
\newcommand{\fulldh}[0]{\CIRCLE}
\newcommand{\partlydh}[0]{\LEFTcircle}
\newcommand{\emptydh}[0]{\Circle}
\newcommand{\possibledh}[0]{$\odot$}
\newcommand{\NAdh}[0]{\emptyset{}}
\newcommand{\R}[1]{\textbf{(R#1)}}
\newcommand{\Rplain}[1]{\textbf{R#1}}
\newcommand{\Sc}[1]{\textbf{(S#1)}}

% -----------------------------------------------
% Commands

%% From https://tex.stackexchange.com/a/225078/17107, by Steven B. Segletes
\newcommand\fauxsc[1]{\fauxschelper#1 \relax\relax}
\def\fauxschelper#1 #2\relax{%
  \fauxschelphelp#1\relax\relax%
  \if\relax#2\relax\else\ \fauxschelper#2\relax\fi%
}
\def\Hscale{.83}\def\Vscale{.72}\def\Cscale{1.00}
\def\fauxschelphelp#1#2\relax{%
  \ifnum`#1=\lccode`#1\relax\scalebox{\Hscale}[\Vscale]{\char\uccode`#1}\else%
    \scalebox{\Cscale}[1]{#1}\fi%
  \ifx\relax#2\relax\else\fauxschelphelp#2\relax\fi}
%% End of stackexchange code


\newcommand{\Cpp}{C\texttt{++}}
\newcommand\Figref[1]{Figure~\ref{#1}}
\newcommand\figref[1]{Figure~\ref{#1}}
\newcommand{\mdm}[1]{\fauxsc{#1}}

\title{Agile Research Data Management with Open Source: LinkAhead}
\subtitle{}
% \shorttitle{Agile RDM}
%
%
% \title and \shorttitle are mandatory, \subtitle can be omitted by just leaving the string empty (please do not comment out the subtitle line)
%
%

%
%
% In order to anonymise your submission, the information on authors and
% affiliations have to be provided in the file: authors.tex
%
% Thanks and acknowledgements  and funding
% information have to be saved in: acknowledgements.tex
%
%

\keywords{Data Management, Research Data Management, Agile Data Management, Software Tools, FAIR
  Data, Good Scientific Practice}
%
%
% 4-6 meaningful keywords that describe the paper



\dataavailability{}

\softwareavailability{Software can be found here: \url{https://gitlab.com/linkahead} and at
  \href{https://doi.org/10.5281/zenodo.7752417}{DOI:10.5281/zenodo.7752417}}
%
%
% Please anonymise your repositories with https://anonymous.4open.science/ or similar services
%
%


\addbibresource{references.bib}
%
%
% Save all your references in this Bib(La)TeX file
%
%


\begin{document}
\maketitle
\begin{abstract}

  % - The background of the research topic
  % - The problem specification or target area
  % - The manner in which the problem was targeted

  Research data management (RDM) in academic scientific environments increasingly enters the focus
  as an important part of good scientific practice and as a topic with big potentials for saving
  time and money.  Nevertheless, there is a shortage of appropriate tools, which fulfill the
  specific requirements in scientific research.  We identified where the requirements in science
  deviate from other fields and proposed a list of requirements which RDM software should answer to
  become a viable option.

  We analyzed a number of currently available technologies and tool categories for matching these
  requirements and identified areas where no tools can satisfy researchers' needs.  Finally we
  assessed the open-source RDMS (research data management system) LinkAhead for compatibility with the
  proposed features and found that it fulfills the requirements in the area of \emph{semantic,
    flexible data handling} in which other tools show weaknesses.

  % Max. 150 words.

\end{abstract}

% Content
% %%%%%%%
%
% - Challenges for RDM
%   - Data lifecycle
% - Proposed features
%   - User acceptance
%     - Automation
%     - Search templates
% - LinkAhead
%   - History
%   - Classification of LinkAhead in relation to relational databases and NoSQL approaches
%   - Architecture
%     - Client libraries
%   - Crawler framework
%   - demo server
%   - documentation
%

\linenumbers

\section{Introduction}\label{intro}

Research units, from small research groups at universities to large research and development
departments are increasingly confronted with the challenge to manage large amounts of data, data of
high complexity\cite{bauer_architecturebiomedical_2016,borgman_conundrumsharing_2012} and changing
data structures\cite{khattak_changemanagement_2013,schneider_ontologiesdata_2020}. The necessary
tasks for research data management include storage, findability and long-term accessibility for new
generations of researchers and for new research
questions\cite{schneider_ontologiesdata_2020,wilkinson_fairguiding_2016,higman_threecamps_2019}.

In spite of the advantages of implementing data management
solutions\cite{hruby_centralizedresearch_2013}, we found a lack of standard methods or even standard
software so far for research data management, especially in the context of quickly evolving methods
and research targets.  We hypothesize that the reason for this deficit is that scientific research
poses unique challenges for data management, since it is characterized by constant innovation, short
lived research questions, trial-and-error approaches, and the continuous integration of new
insights.

We propose \emph{agile research data management} as a promising approach to meet the special
requirements of scientific research and to fully leverage the benefits of increased research
digitalization, automated data acquisition methods and storage capabilities.

For this article, we identified the specific challenges for research data management (RDM) and
defined eleven requirements which suitable RDM software should have to (a) fulfill the practical
needs and (b) be accepted by the potential users.  We then matched existing tools against these
requirements and found areas where the tools show substantial need for improvement.

Finally we present the LinkAhead\cite{_linkahead_2023,fitschen_caosdbresearch_2019} toolkit as a viable approach
to satisfy all the proposed requirements.

\section{Challenges for research data management}\label{sec:chall}

\subsection{The scientific data lifecycle: the need for proper tooling}\label{sec:chall:lifecycle}

Data which accrues in scientific research is more than just experimental readings, field notes or
interview recordings.  In order to fully represent the research journey and eventually enable
reproducible science, the data from every research step may become relevant.  We identify the
challenges to make this data usable in a way that leads to reproducible, and time-efficient,
research.

\begin{figure}[tb]
  \centering
  \includegraphics[width=0.7\textwidth]{figures/data_lifecycle_anonymous.pdf}
  \caption{\label{fig:lifecycle} Schematic illustration of the scientific data lifecycle. Data can
    be obtained from every step, and in most cases the relationship between data entities is just as
    relevant as the raw data.  Blue thick arrows denote the direction in which information flows in
    normal research.  Thin black arrows indicate data flow to and from a research data management
    system.  While this example focuses on experimental and laboratory centered disciplines,
    comparative lifecycles also exist for theoretical sciences and most fields in the humanities.
    }
\end{figure}

\Figref{fig:lifecycle} shows a schematic of different research steps during the research lifecycle,
during which important data is generated.  For full reproducibility, it is not sufficient however to
simply store any data that one acquires, but also to represent the semantic connections and make
these connections searchable.

In more detail, the most relevant sources and targets for data in scientific research are (numbered
from (D1) to (D6)):

\begin{description}
\item[Prior publications (D1)] An important part of good scientific practice (GSP) is to credit the
  influence of prior work, written by the scientists themselves or third parties.  Linking one's own
  work to previous publications --- articles or published data from repositories --- and making
  these connections public helps to assess reproducibility and may lead to fruitful data-reuse in
  unforeseen contexts.\cite{radicchi_citationnetworks_2012}  An RDMS should be able to trace back each
  data item to previous scientific publications on which it is based.
\item[Ideas and SOPs (D2)] The data here consists mostly of text documents which describe thoughts,
  hypotheses and planned standard operating procedures (SOPs).  These documents fill the gap between
  previous work and the next round of data acquisitions, they often also work as a blueprint for the
  data acquisition phase.\cite{manghani_qualityassurance_2011} A scientist may consult their RDMS to answer
  questions like ``Which SOP was used when experiment X was carried out to generate data file Y?''.
\item[Lab data (D3)] Environmental data, device settings, used SOPs and ingredients and other
  incidental data typically accrues during the course of experiments and was traditionally stored in
  paper laboratory notebooks.  Currently, a lot of laboratories switch to electronic lab notebooks
  (ELNs) for the same purpose.  While this data is often seen as second-class ``metadata'', we hold
  that since often conclusions can be drawn from it, it deserves the same handling as final
  instrument
  readings.\cite{abujayyab_biologicaltreatment_2022,nicholson_analysismetadata_2020,griss_identifyingnovel_2015}

  During work in the lab, software must be as unintrusive as possible, with efficient user
  interfaces.
\item[Experimental results (D4)] These are what is often considered the \emph{main} data.  For
  meaningful analysis, data from experimental results mostly must be enriched with additional data
  from experimental or device settings or from processed samples, to filter for special conditions,
  to compare settings or to verify that values are compatible with standard
  literature.\cite{silvello_semanticrepresentation_2017}
\item[Numerical simulations (D5)] Similarly to experimental results, data obtained from numerical
  procedures can not be interpreted without knowledge about used software and parameters, possibly
  hardware conditions and input from laboratories or third-party data
  sources.\cite{fitzpatrick_issuesreproducible_2019} Since bit-for-bit reproducibility is possible
  in theory, all relevant settings should be stored unchanged.
\item[Data analysis (D6)] When analyzing data from previous steps, storing not only the used
  programs, scripts, and their parameters, but also the semantic connections enables later
  researchers to reconstruct which method was used, which assumptions were made and under which
  conditions the input data was
  gathered.\cite{poldrack_computationalinformatic_2019,strozzi_scalableworkflows_2019}
\item[Next publication (D1)] Formally the end of the lifecycle, but of course also the beginning of
  many new ones, a publication contains a number of statements which are supported by data from
  previous steps. A comprehensive RDMS could quickly answer a question like ``In figure X, which
  methods were used to analyze the data, which devices and software were used to acquire the raw
  data, and which assumptions were made when planning the experimental setting?''
\end{description}

This list focuses on experimental and laboratory centered disciplines like engineering or natural
sciences, but of course in the humanities and theoretical sciences, there are equivalent steps which
are equally important to preserve and link to each other.

\subsection{Specifics of scientific research data management}\label{sec:chall:specifics}

There are some needs for data management which are specific to or more pronounced in scientific
research, which we will label by (S1) through (S5):

\begin{description}
\item[Interoperability] Scientists tend to work with their own custom-written
  software\cite{hannay_howscientists_2009,segal_whensoftware_2005,wilson_softwarecarpentry_2006},
  which often requires files with data to be directly accessible to the OS via a file system \Sc{1},
  remote or locally.  Also programmatic access (query, retrieve, update) to data via network APIs
  \Sc{2} is a necessity for many scientific data uses.
\item[Agility] Traditional DMS require users to define a data model and stick to
  it\cite{anderson_issuesbiomedical_2007}.  All data to be entered has to conform to the data model as it was
  defined.  Research however is defined by having undefined outcomes, the research questions,
  experimental setup or analysis methods change more often than not over the course of one
  investigation.\cite{bron_mediastudies_2016} We therefore identify \Sc{3} as the special need for
  flexibility regarding the data model.
\item[Learning curve] Scientific research is founded upon the contribution of many participants,
  with different qualifications, varying research foci and high fluctuations.  As a consequence, a
  steep learning curve for using an RDMS would be detrimental to its adoption \Sc{4}.
\item[Early usefulness] Systems which only store data, but do not provide short-term advantages,
  have high acceptance barriers.  Especially in academic research, junior scientists with short-term
  contracts have little incentive to invest time and money in systems which only may pay out on
  longer timescales.\cite{anderson_issuesbiomedical_2007} Therefore, RDMS should offer some tangible
  advantages on the short run \Sc{5}.
\end{description}


\section{Requirements for a scientific RDMS}\label{sec:requirements}

Based upon the challenges from the previous section, we propose a set of requirements for an
RDMS to be a useful tool for scientific research.

\subsection{General requirements}\label{sec:requirements:general}

\begin{description}
\item[\R{1} Semantic linkage] In order to retain the semantic context in which data is
  embedded, it must be possible in the RDMS to link data sets with each other in a meaningful way,
  i.e., the links must bear some meaning.  The default linking possibilities and properties of the
  data types in the RDMS form the \emph{data model}.
\item[\R{2} Flexible data model] Researchers require an RDMS for structured storage of data, where
  the data model can be changed on the fly, without the need to migrate or discard existing data
  (\Sc{3}).  When the data model is changed, for example due to new machines, protocols or evolving
  research questions, the existing data must remain valid and usable.  A change in ontological
  semantics \emph{now} must be compatible with previous semantics \emph{then}.
\item[\R{3} Searchability] The RDMS should have easily accessible search options not only for
  property values of stored entities, but also for links to other entities and properties (and link)
  thereof.  This deep search allows the traversal of the structured knowledge graph and delivers
  actual utility value.
\item[\R{4} Sustainability] In order to assure long-term access to stored data, software solutions
  must have some safeguard against becoming unmaintained.  This could be achieved by being either
  open-source software or ``too big to fail''.  In the case of open-source software, either the
  community or other companies could step in, if the original maintainers stopped their support.  On
  the other hand, if a software system is very widely adopted and thus indispensable, it is unlikely
  to be abandoned or left unsupported.
\item[\R{5} Open APIs] For interaction with third-party programs, the RDM must have an API with low
  entrance barriers (\Sc{2}). In research contexts, these third-party programs are often
  custom-written by scientists without explicit computer science background, so extensive
  documentation of the API is very desirable.
\end{description}

\subsection{Automation}\label{sec:requirements:automation}
% crawler, data repos, lab workflows, ELNs

Automation of repetitive data integration reduces error rates and frees users to concentrate on more
challenging tasks.  It is therefore desirable for an RDMS to have:
\begin{description}
\item[\R{6} Synchronization] The RDMS should make it easy for its administrators to integrate
  existing data sources (for example databases or file systems with structured folder hierarchies)
  into the RDMS:\@ The RDMS should be synchronized automatically with data from these sources, which
  makes these data available in a unified manner via the RDMS interface.  Note that the RDMS can not
  solve the conceptual problem of a single source of truth when synchronizing data from different
  sources, but it can at least highlight potential conflicts and where they first occurred to
  administrators.
\item[\R{7} ELN integration] Research work in the lab is increasingly documented with electronic lab
  notebooks (ELNs)\cite{kanza_electroniclab_2017,higgins_considerationsimplementing_2022}, which
  allow to conveniently enter device and experimental settings in a semi-structured way.  This data
  is usually critical in the analysis of acquired raw data from instruments, e.g., for searching
  specific data sets or filtering by parameters.  There should be a possibility that the RDMS
  integrates the ELN data and presents it like data from other sources.
\item[\R{8} Workflow representation] While following one SOP, the laboratory workflow is often highly
  standardized, which makes it suitable for representation within the RDMS.\@  The RDMS should support
  workflows with different states, which can only be switched in an admin-defined pattern.  This
  simplifies the work for users, because they may e.g., only see the interfaces which are relevant
  for the current sample processing step.
\end{description}

\subsection{Specific requirements for scientific work}\label{sec:requirements:scientific}

As introduced in section \nameref{sec:chall:specifics}, some requirements arise from scientific
research specifically.

\begin{description}
\item[\R{9} Versioning] Mistakes during data acquisition happen, and it must be possible to correct
  existing data sets.  At the same time, this editing must be made transparent and the history of
  each data set must be kept for future inspection.
\item[\R{10} File system integration] For interaction with third-party programs, raw data files must
  be available on standard file systems (\Sc{1}).  Ideally the scientists' workflows should remain
  unchanged by the RDMS.\@
\item[\R{11} Gentle learning curve, early pay-off] To accommodate for the short employment lifecycles
  in science, RDMS should offer straightforward and simple to learn usage possibilities which give
  some early sense of achievement (\Sc{4}, \Sc{5})\cite{abdullah_investigatinginfluence_2016}.  One
  example could be simplified search options which help users understand that an RDMS will make
  their work easier when handling with data.
\end{description}

\subsection{Relation to FAIR data management}

FAIR data management is seen as a general requirement by the scientific community at large.  We hold
that a research data management system fulfilling \R{1} -- \R{11} can enable research groups to
implement a FAIR data management.

Specifically, \emph{Findability} can be achieved because each data set and collections of data can
be assigned persistent identifiers, data and metadata can be intimately connected and data can be
found through the search functionality of the RDMS.\@

Scientific RDMS can enable \emph{Accessibility} through open and standardized APIs and separation of
raw data and metadata.  RDMS allow for \emph{Interoperability} when users can incorporate existing
ontologies for data model, descriptions and references between data sets.  \emph{Reusability} is
fostered by rich data models including licenses, provenance information and which follow the
respective communities' standards.

\section{Current state of the tools landscape}\label{sec:tools}

We give a short overview over existing solutions, tools and approaches and over their possibilities.
We also classify the extent to which they cover the required features.

\subsection{Technologies and approaches}\label{sec:tools:tech}

Currently, DMS tools exist for a range of fields and use numerous technological and methodological
approaches.  Different sources use different definitions for some of the following categories, so we
try make our definitions explicit, where necessary.

\begin{description}
\item[ELNs] Here we use the definition of Harvard Medical Scool\cite{hms_eln_2023}:
  \begin{quote}
    An Electronic Lab Notebook (ELN) is a software tool that in its most basic form replicates an
    interface much like a page in a paper lab notebook. In an ELN you can enter protocols,
    observations, notes, and other data using your computer or mobile device.
  \end{quote}

  Electronic laboratory notebooks replace paper-based physical solutions to document the scientific
  workflow in laboratories, but also partly planning and analysis of obtained data.  For the sake of
  this article, ELNs are distinct from other lab-oriented data management software in that ELNs
  focus on the user experience while entering data in a laboratory environment and allow to enter
  data in a semi-structured way, often much like a text editor with the possibiliy to add a number
  of structured fields.  The structure can sometimes be defin by means of user editable templates.
  Typical examples are eLabFTW\cite{carpi_elabftwopen_2017},
  Chemotion\cite{tremouilhac_chemotioneln_2017}, RSpace\cite{_rspaceeln_},
  eLabJournal\cite{_elabjournal_} and other\cite{kanza_electroniclab_2017}.
\item[Field-specific solutions] Many scientific field have specialized data management solutions for
  their fields which cater to the specific needs, such as chemical structure searches, material
  property tables, sample management or domain specific data visualization.  Often, these solutions
  excel in their purposes but customization options or interaction possibilities may be
  limited. Examples are Nomad\cite{draxl_nomadlaboratory_2019}, C6H6.org\cite{patiny_c6h6nmr_2018},
  Chemotion\cite{tremouilhac_chemotioneln_2017,}, JuliaBase\cite{bronger_juliabase_2023} among others.
\item[Data, article and software repositories] Most scientific journals and some funding agencies
  require scientists to publish the data underlying their publications in a publicly accessible data
  repository.  There are data repositories with custom software, and an increasing number of public
  repository instances using off-the-rack software like
  Dataverse\cite{king_introductiondataverse_2007}, Invenio\cite{caffaro_inveniomodern_2010},
  DSpace\cite{smith_dspaceopen_2003} or CKAN\cite{winn_opendata_2013}.  Data repositories cover
  \textbf{(D1)} in the data lifecycle and offer some search functionality, in all but very few cases
  they are intended for immutable data at the time of publication.  Data models range from very
  simple (only authors and text description) over completely user defined key-value pairs to domain
  specific fixed data models for domain repositories.

  Similarly, software and articles are stored in specialized repositories, which often have
  extensive metadata capabilities for the entities stored within them.
\item[Data storage systems] Data storage is a necessary prerequisite for scientific research and
  thus there are many well established systems: mirrored network file systems (e.g., NFS, CIFS) with
  regular backups to tape archives on the one hand and object stores (e.g., S3) on the other hand,
  which store binary blobs outside classical file system structures.
\item[SQL databases] Plain SQL databases use tables where rows represent records and columns
  represent the data sets' attributes or properties.  Each table with a fixed set of columns of
  mostly fixed types represent one type or class of data, defining the properties available for that
  type.

  Because SQL databases are readily available and can be integrated into most programming languages,
  they are often used as the technical base for both self-written ad-hoc data management solutions
  and existing commercial data management systems
  alike\cite{ailamaki_managingscientific_2010,nourani_clinicaltrial_2019}.
\item[Key-value stores] A contrasting approach to SQL databases (therefore categorized as NoSQL
  databases, popular examples are CouchDB or MongoDB), key-value stores manage data as a collection
  of key-value pairs.  They trade the structure of the SQL paradigm for flexibility, allowing users
  to store whatever they deem appropriate.
\item[RDF, SPARQL] A common concept from academic knowledge representation research, RDF (resource
  description framework)\cite{hartig_rdfconcepts_2023} is a framework and representation standard for
  subject-predicate-object triples.  It has found adoption in the standardization community and some
  applications.  SPARQL is a query language for accessing RDF data and used by knowledge services
  such as Wikidata.\cite{harris_sparqlquery_2013,vrandecic_wikidatafree_2014}
\end{description}

We would also like to mention that some solutions incorporate one or more if these approaches as
components.  For example Kadi4Mat\cite{brandt_kadi4matresearch_2021} and Nomad have ELNs as part of
the overall software package.

\subsection{Do existing tools meet the requirements?}\label{sec:tools:requirements}

We discuss to what degree these technologies and tools are able to fulfill the requirements \R{1} --
\R{11} listed above.  Here we differentiate between technologies, which may be used when
implementing applications on the one hand, and tools on the other hand which are candidates for data
management solutions.

\subsubsection{Technologies}

\begin{description}
\item[RDF, SPARQL] RDF was designed and is well suited to represent semantic relationships between
  entities and local RDF collections can be extensively searched with SPARQL by trained
  experts. There is a number of standardized RDF serializations which can be generated and read by a
  many programming languages.  Data models can be implemented using \emph{RDF Schema}, which is
  based upon RDF.  Entities can reference entities located on other instances, which brings greater
  flexibility, but raises issues about data mutability and searchability.
\item[SQL databases] Relational databases thrive on relations between tables and thus allow some
  degree of semantic linking, albeit with very limited flexibility.  Searching is possible, but
  requires a certain degree of expertise, which can be mitigated by external helper tools.  There
  are standardized implementations, open source and proprietary alike, which can be expected to
  continue for the foreseeable future.
\item[Key-value stores] NoSQL databases allow users a comprehensive degree of freedom when storing
  data, but at the same time often provide no overarching structure to enforce certain data model
  properties.  Semantic linkage thus often is limited to convention instead of internalized
  structures.  Searchability is comparable to traditional SQL databases, and there is a large number
  of implementations.
\item[Data storage systems] As a basic technology to store raw files or objects, data storage
  systems do not have the ability to link data or provide a data model.  Searching data for
  associated metadata or file content is possible for some storage systems.  Higher-level
  functionality is not available within the data storage systems themselves.
\end{description}

These base technologies have in common that they mostly do not provide functionality such as
high-level network APIs, graphical user interfaces, integration with other components or versioning.
Also they target technical audiences and thus feature steep learning curves for data manipulation
and searching alike.

\begin{table}[tb]
  % \centering
  \begin{minipage}{\textwidth}
  \begin{NoHyper} % Against hyperref warning about footnote
  \begin{tabular}{llllllllllll}
    \toprule
    Technology & \Rplain{1} & \Rplain{2} & \Rplain{3} & \Rplain{4} & \Rplain{5} & \Rplain{6}
    & \Rplain{7} & \Rplain{8} & \Rplain{9} & \Rplain{10} & \Rplain{11} \\
    \midrule
    %                | sem. link   | flex. DM    | search      | sustain    | open API   |
    %        synchr. | ELN     | workfl. | versioning  | file syst | simple |
    RDF + SPARQL     & \fulldh{}  & \fulldh{}  & \fulldh{}  & \fulldh{} & \fulldh{} %
        & \emptydh{} & \NAdh{} & \NAdh{} & \emptydh{}  & \emptydh{} & \emptydh{} \\
    SQL              & \fulldh{}  & \emptydh{}  & \fulldh{}  & \fulldh{} & \fulldh{} %
        & \emptydh{} & \NAdh{} & \NAdh{} & \emptydh{}  & \emptydh{} & \emptydh{} \\
    Key-value stores & \partlydh{} & \partlydh{} & \partlydh{} & \fulldh{} & \fulldh{} %
        & \emptydh{} & \NAdh{} & \NAdh{} & \emptydh{}  & \emptydh{} & \emptydh{} \\
    Data storage     & \emptydh{}  & \emptydh{}  & \partlydh{} & \fulldh{} & \fulldh{} %
                     & \NAdh{}    & \NAdh{} & \NAdh{} & \partlydh{} & \fulldh{} & \partlydh{} \\
    \midrule
    Tools & \Rplain{1} & \Rplain{2} & \Rplain{3} & \Rplain{4} & \Rplain{5} & \Rplain{6}
    & \Rplain{7} & \Rplain{8} & \Rplain{9} & \Rplain{10} & \Rplain{11} \\
    \midrule
    ELNs & \partlydh{} & \partlydh{} & \partlydh{} & \fulldh{} & \fulldh{} & \partlydh{}
         & \NAdh{} & \fulldh{} & \fulldh{} & \partlydh{} & \fulldh{}\\
    FSS\footnote{field-specific solutions} & \partlydh{} & \partlydh{} & \partlydh{} & \partlydh{} & \partlydh{}
                       & \possibledh{}
                       & \possibledh{} & \fulldh{} & \partlydh{} & \partlydh{} & \fulldh{}\\
    Repositories & \emptydh{} & \emptydh{} & \partlydh{} & \fulldh{} & \partlydh{} & \possibledh{}
    & \emptydh{} & \emptydh{} & \fulldh{} & \emptydh{} & \fulldh{}\\
    \bottomrule
  \end{tabular}\\[-0.5em]
  \end{NoHyper}
  \end{minipage}\\[1em]

  \begin{tabular}{llp{2em}ll}
    \toprule
    % Requirements & \\
    \Rplain{1} & Semantic linkage & & \Rplain{7} & ELN integration \\
    \Rplain{2} & Flexible data model & & \Rplain{8} & Workflow representation \\
    \Rplain{3} & Searchability & & \Rplain{9} & Versioning \\
    \Rplain{4} & Sustainability & & \Rplain{10} & File system integration \\
    \Rplain{5} & Open APIs & & \Rplain{11} & Gentle learning curve, early pay-off \\
    \Rplain{6} & Synchronization \\
    \bottomrule
  \end{tabular}
  \caption{Data technologies, tools and if they meet the requirements.\\Symbols used: \fulldh{}:
    yes, \emptydh{}: no, \partlydh{}: partly, \possibledh{}: may be possible to implement, \NAdh{}:
    not applicable.\\Note that a ``\partlydh{}'' may signify that not all particular examples of the
    category fulfill the requirement, but it may also mean that (nearly) all examples fulfill parts
    of the requirement.}
\end{table}\label{tab:tools}


\subsubsection{Tools}

\begin{description}
  % Source: https://eln-finder.ulb.tu-darmstadt.de/search
\item[ELNs] ELNs target at a non-technical audience and thus generally aim to have low entrance
  barriers, with tutorials and graphical help functions.  Most generic ELNs allow basic linking
  between stored records and searches thereof, and users are guided in their work of entering data
  by means of templates.  These templates often do not have a semantic meaning however, but serve
  only as a means of suggesting data fields.  Data is organized around lab sessions, the main
  datatype are notes from the laboratory.  ELNs only started to become the de-facto standard in
  laboratories over the last decade, so the market is far from settled.  There are open-source and
  proprietary software solutions, by large players and by solo enterprises.  Nearly all ELNs
  developed over the last five years now offer APIs for third-party access, and many allow users to
  organize their workflows, such as different processing steps for a sample.

  Synchronization with other data sources or integration with file systems is not a core element of
  ELNs and as such rarely seen.  Similarly, synchronization with other data sources exists only on a
  case-to-case base.  Versioning of stored entities is possible to some extent for most ELNs.
\item[Field-specific solutions] Semantic linking may be possible to a certain amount as permitted by
  the data model, which typically is limited to the use cases foreseen by the developers. Similarly,
  searching the data often is limited to key-value filters on the specialized data types.  Some
  solutions (e.g., NOMAD) implement their own ELNs, but integration with third-party ELNs and
  synchronization with other data sources does not exist generally: it could be implemented via
  APIs, in those cases where they exist. Support for workflows is generally quite good, and the
  learning curves are adapted to the audience.  Versioning of data and integration of existing file
  systems may be present in some systems.  Long-term availability of software support may be an
  issue when these solutions are only developed by a small set of people or even individuals, often
  in time-limited funding situations.  In these cases, open-source software can be an insurance for
  the future, especially if there is sufficient development documentation.
\item[Repositories] Data repositories only cover a small subset of data management use cases and as
  such generally do not implement many of the requirements.  They may allow semantic linkage between
  entities, but do have encompassing data models at all.  Searching is limited to key-value filters
  and full-text, sometimes referenced data sets can also be used as filters, but there may be APIs
  which allow external tools to improve on this shortcoming.  Repositories generally have
  institutional funding so that long-term availability can be seen as guaranteed.  Synchronization
  with other data sources, local file system or ELN integration or workflow representation does not
  make sense, since repositories are meant for manual data archive upload at the end of the
  scientific life cycle.  Upload of data to archives is very straightforward in most cases, and
  editing of uploaded data does not invalidate the original version, but only marks it as out of
  date.
\end{description}


\subsubsection{Summary of existing tools}\label{sec:tools:summary}

The requirements coverage of the examined technology and tool classes are shown in table\
\ref{tab:tools}.  We see that while existing tools cover a wide range of the required features,
there are significant shortcomings in two areas: flexible data models, semantic linkage and
searchability on the one hand, and integration with ELNs, other devices and file systems on the
other hand.  Due to the large number of available products, for each requirement, there are ELN and
field-specific solutions which may fulfill it at least partly, although such a product in general
does not cover all requirements.

We stated earlier that these topical fields are especially relevant in scientific research.  As an
effect DMS have been widely successful in many areas such as finance, administration, and high-tech
industries\cite{raptis_datamanagement_2019,tuyl_investigationnonacademic_2018}, but remain scarce in
both academic and private sector
research\cite{tuyl_investigationnonacademic_2018,kennan_researchdata_2015}.  In summary, we find the
need for a tool which fills the requirements for semantic, flexible data management and has
sufficient synchronization and ELN integration capabilities.

\section{LinkAhead}\label{sec:linkahead}

We hold that LinkAhead\cite{fitschen_caosdbresearch_2019}, an agile data management framework,
fulfills the proposed requirements from section \nameref{sec:requirements}. LinkAhead was initially
developed under the name ``CaosDB'' by one of our colleagues, Timm Fitschen, during his time at the
Max Planck Institute for Dynamics and Self-Organization, and others.\cite{linkahead_humansmd_2023}
In 2018, LinkAhead was released, still as ``CaosDB'' under the AGPLv3 license on
gitlab.com.\cite{_linkahead_2023} Since 2020, LinkAhead has found increased adoption in multiple
research facilities.

In this section, we first describe LinkAhead in detail, then we assess to which extent LinkAhead fulfills
the proposed requirements and finally we give an overview over limitations and possible enhancements
in the future.

\subsection{Detailed description}\label{sec:linkahead:desc}

LinkAhead was developed out of the need for a data management solution that can cope with large data
amount from automated sources and from existing file systems alike and that allows researchers to
quickly adapt the way how data sets are connected or described.  These needs reflect on the design
choices which were taken over the course of development.  LinkAhead is a general research data
management system: specialized solutions such as ELNs, sample management systems, document
management systems or other can be developed on top of it, according to specific needs.


\subsubsection{Data Model}\label{sec:datamodel}

\begin{figure}[tbp]
  \centering
  \includegraphics[width=0.75\textwidth]{figures/meta_data_model/meta_data_model.pdf}
  \caption{\label{fig:mdm}The metadata model of LinkAhead.}
\end{figure}

LinkAhead's \emph{meta} data model is shown schematically in \figref{fig:mdm}.  The base type for
everything is \mdm{Entity}, with the inheriting types \mdm{Property} (attributes of \mdm{Entities},
may be list values and references to other \mdm{Entities}), \mdm{RecordType} (templates for actual
data sets) and \mdm{Record}.  Actual data is typically stored in \mdm{Record}s, which \emph{inherit}
from one or more \mdm{RecordType}s and thus have all the \mdm{Properties} defined therein.  The
\mdm{RecordType}s may form a complex inheritance hierarchy themselves.  \mdm{File} entities are
similar to Records, but additionally are connected to files which may reside on conventional file
systems or potentially in abstracted cloud storage systems.  This approach to use files at their
current locations instead of duplicating file content not only increases LinkAhead's scalability, but
also lower the entrance barrier, since scientists can access the managed file in their traditional
ways.

Details of this metadata model in LinkAhead are elaborated on in~\cite{fitschen_caosdbresearch_2019},
but it should
be clear now already that LinkAhead provides the \emph{Semantic linkage} feature.

%   - Classification of LinkAhead in relation to relational databases and NoSQL approaches

In LinkAhead, the \emph{data model} of the stored data refers to the \mdm{RecordType}s and their
\mdm{Properties}, which together describe the pattern to which newly created data sets should
conform.  The data model in LinkAhead can be modified at any time, but the changes only take effect for
data to be inserted \emph{after} this modification.  Existing data is not affected and remains
unchanged.  This property fulfills the proposed \emph{Flexible data model} feature.

\mdm{Properties} of \mdm{RecordTypes} are allocated a graded \emph{importance}, which denotes if
this \mdm{Property} is either \emph{obligatory}, \emph{recommended} or merely \emph{suggested} for
\mdm{Records} which inherit from this \mdm{RecordType}, when a user creates a new \mdm{Records}.
This system of importances and the fact that \emph{legacy} data is not necessarily consistent with a
\emph{modified} data model was a deliberate design decision.  The rationale was that when the data
model changes, the meaning at the time of data creation should have priority over consistency with
later data models.

This possibility to completely change the data model, while not giving up on a general structure,
places LinkAhead between traditional SQL based relational databases and NoSQL approaches (c.f.\
\figref{fig:paradigms}).  While we described above why rigid SQL databases are not suited for use in
dynamic research environments, giving no structure (the NoSQL paradigm) tends to lead to incoherent
data which is hard to search.  A common implementation of NoSQL approaches in the context of data
management are \emph{data lakes}, where raw data can be stored and annotated with metadata. The
missing structure in Data Lakes however has lead to the tongue-in-cheek colloquialism ``Data
Swamp''.  A third approach, using graph databases to represent semantic information, has not found
its way into general adoption to our knowledge, presumably because the query languages tend to
become very unwieldy, compare the appendix~\nameref{app:sparql} for an example.

\begin{figure}[tb]
  \centering
  \includegraphics[width=0.55\textwidth]{figures/linkahead_paradigm_advantages.pdf}
  \caption{\label{fig:paradigms} LinkAhead compared to other database approaches.}
\end{figure}

\subsubsection{Architecture and Libraries}
\label{sec:arch}

\begin{figure}[htb]
  \centering
  \begin{subfigure}{0.31\textwidth}
    \includegraphics[width=\textwidth]{figures/linkahead_architecture.pdf}\phantomsubcaption{\label{fig:arch:arch}}
  \end{subfigure}
  \hfill
  \begin{subfigure}{0.6\textwidth}
    \includegraphics[width=\textwidth]{figures/linkahead_crawler.pdf}\phantomsubcaption{\label{fig:arch:crawler}}
  \end{subfigure}
  \caption{\label{fig:arch} \textbf{(a)}~LinkAhead's server-client architecture with client libraries
    and backend components.  Dotted elements are under development.  \textbf{(b)}~The crawler
    framework facilitates fast development of custom data integration from a diversity of sources.}
\end{figure}

%   - Architecture
%     - Client libraries

\sc{LinkAhead} uses a client/server based architecture, as depicted in \figref{fig:arch:arch}. LinkAhead
has is a REST API for simple access by traditional clients and a web interface for browsers, as well
as a gRPC API which allows for more complex operations, such as atomic content manipulations.  The
existing client libraries\footnote{A list of the available libraries with the respective source code
  repositories are given in the Appendix section~\nameref{app:list}.} and the open APIs provide the
proposed \emph{Interoperability} requirement.

%   - Crawler framework
One particularly useful client library component is the \emph{LinkAhead Crawler} framework.  This
extensible framework simplifies the work to synchronize external data sources with LinkAhead through a
plugin system.  The crawler workflow can be characterized as follows:

\begin{enumerate}
\item The crawler checks its data sources for new or changed data stores, such as file systems or
  the content of other databases.  This may happen periodically or be triggered manually by users.
\item Each new data source is fed to a so-called \emph{CFood plugin} for consumption.  There is a
  choice of existing plugins, or administrators can write their own.  The CFood plugin's job is to
  build LinkAhead entities from the consumed data and to specify \emph{Identifiables}, which work as
  search patterns.  Administrators can mostly define simple CFood plugins by YAML configuration
  files\cite{tutorial_crawler_2023} which is a more user-friendly approach than for example the
  mappings defined by the W3C's R2RML standard.\cite{_r2rmlrdb_2012}
\item The crawler checks for each \emph{Identifiable} if a corresponding entity exists already in
  LinkAhead.  If there is no corresponding entity, the entity as returned by the CFood plugin is
  inserted into LinkAhead.  If there is already an existing entity, the Crawler will attempt to merge
  the existing with the new entity and notify the data curators in case of merge conflicts.
\end{enumerate}

This tool set provides the \emph{Synchronization} requirement, and if ELNs are used as external
data source, the \emph{ELN integration}.  Practical use of LinkAhead crawler framework has previously
been demonstrated in~\cite{spreckelsen_guidelinesstandardized_2020} and ELN integration was
implemented as a working proof-of-concept in~\cite{tomworden_linkaheadcrawler_2023}.

\subsubsection{Miscellaneous features}\label{sec:features}

% - deep search
% - search templates
% - versioning
% - workflow with state machine

\begin{description}
\item[Deep search] LinkAhead offers a simple semantic query language, which borrows some semantics from
  SQL, but has a focus on usability for non-technical users.  The LinkAhead query language makes deep
  search easy with expressions like the following:

  \texttt{FIND Analysis WITH quality\_factor > 0.5 \\\mbox{}\hspace{16ex}AND WITH Sample WITH weight < 80g}

  This convenient nesting of query expressions circumvents the \texttt{JOIN} operations from
  traditional SQL languages.  A full documentation of LinkAhead's query language is available
  online\cite{_linkaheadquery_2023} and in LinkAhead's sources.
\item[Search templates] LinkAhead's web interface provides customizable search templates which allow
  more advanced users to create their own query templates, which can then be shared with novice
  users for \emph{simplified searches}.  In query templates, users can insert custom strings into
  pre-defined locations of a search query, see \figref{fig:query_template}.
\item[Versioning] When entities are modified in LinkAhead, time and user of the change are recorded and
  LinkAhead puts the previous version onto a history stack and amends the current version with link to
  the previous version.  Over time, each entity may thus grow to a tree of linked versions, which
  can be retrieved via the web UI or programmatically through the APIs.  This feature of LinkAhead
  enables scientific research data management users to adhere to the principles of good scientific
  practice.
\item[State management] In LinkAhead, users may declare a state machine of states and allowed
  transitions.  Users may then affix states to entities, and these states can then only be changed
  according to the rules of the state machine.  In this way, users can implement a \emph{workflow
    representation} which ensure that for example laboratory samples run through a specified list
  of preparation steps in order.
\end{description}

\begin{figure}[htb]
  \centering
  \includegraphics[width=0.9\textwidth]{figures/query_template/query_template.png}
  \caption{\label{fig:query_template}A query template in LinkAhead's web UI.\@ The user can enter a
    custom value into an input field \ding{192} and the template is then executed as a plain LinkAhead
    query \ding{193}.  Screenshot from \url{https://demo.indiscale.com}.}
\end{figure}

\subsubsection{Availability and documentation}\label{sec:availability}

% - gitlab
% - demo server
% - docker containers at IndiScale
% - documentation

LinkAhead is available on the public Git repository gitlab.com at \url{https://gitlab.com/linkahead}, a
detailed list of LinkAhead's sub projects is given in the annex.  LinkAhead's source code is licensed
under the AGPLv3 (Affero GNU Public License, version 3).  Community contribution workflows, a code
of conduct and general development guidelines are outlined in
\url{https://gitlab.com/linkahead/linkahead-meta} and in the sub project specific code repositories.  The
community chat\cite{_linkaheadmatrix_2023} is currently populated with 33 members, the contributors
file lists 19 active contributors\cite{linkahead_humansmd_2023}.

For the interested public, there is a live demo server at \url{https://demo.indiscale.com}, hosted
by IndiScale GmbH.  IndiScale GmbH also provides commercial support, development and
customization services for LinkAhead.  There are also Debian/Ubuntu packages to run precompiled
LinkAhead for download at \url{https://indiscale.com/download}.

LinkAhead's sub projects each have their own documentation in their source directories.  The
documentation is also available online at \url{https://docs.indiscale.com}.

\subsection{Requirements matching}\label{sec:linkahead:requirements}

In the following list, we evaluate if and how LinkAhead matches the requirements proposed in section
\nameref{sec:requirements}:

\begin{description}
\item[\R{1} Semantic linkage] Links between \mdm{Entities} in LinkAhead are implemented as reference
  typed \mdm{Properties}, these \mdm{Properties} can be restricted to Entities with certain parents,
  adding an additional ontological level.  All \mdm{Properties} can have a description and higher-order
  properties and thus can fulfill the requirements for typical predicates in
  subject-predicate-object relationships in predicate logic oriented triple stored such as RDF.\@
\item[\R{2} Flexible data model] In LinkAhead, the data model, i.e., the set of \mdm{RecordTypes} can be
  modified at any time.  Existing \mdm{Records} are not affected by these modifications and keep
  their properties and inheritance information.
\item[\R{3} Searchability] LinkAhead's query language allows to deeply search the available data for
  simple key-value relations and also for nested relations on the knowledge graph and the related
  entities' properties.
\item[\R{4} Sustainability] LinkAhead is fully open-source and freely available on gitlab.com, with
  options for commercial support.
\item[\R{5} Open APIs] The REST and GRPC APIs included in LinkAhead enable interaction with scientists'
  custom-written programs.  Additionally the existence of client libraries simplifies the usage by
  programmers without formal software development training.
\item[\R{6} Synchronization] LinkAhead's \emph{crawler} framework simplifies the synchronization
  between existing data sources and the RDMS and allows to make a diversity of data accessible at a
  single resource.
\item[\R{7} ELN integration] The LinkAhead crawler may use ELNs as a data source, thus integrating the
  content acquired by ELNs into the RDMS.\@  This makes ELN data searchable and usable equivalently
  to data from their sources.
\item[\R{8} Workflow representation] The state machine in LinkAhead can be used to represent
  standardized workflows.  For example laboratory samples or interview partners or publications may
  have a state whose possible transitions and conditions can be specified.
\item[\R{9} Versioning] Entities in LinkAhead are versioned and previous content may be displayed and
  recovered.  The content history of entities is stored: which user changed what value at which
  time.
\item[\R{10} File system integration] LinkAhead does not make copies of data files but only references
  the file locations.  The file path or resource identifier is returned upon queries, so that users
  can use the location in their accustomed software.
\item[\R{11} Gentle learning curve, early pay-off] Search queries in LinkAhead can be made more
  accessible to users by templates where only specific values need to be filled in.  The agile data
  model allows scientists to start with a structured data management without the need to develop a
  seemingly overwhelming master plan for their data.  Instead they can start small in an area where
  they expect the most immediate benefits such as improved findability of linked data, and grow the
  data management at a later time.
\end{description}

We find that LinkAhead fulfills the requirements \R{1}--\R{5}, \R{9}--\R{11} ``out of the box'' and
that \R{6}--\R{8} (synchronization, ELN integration and workflows) can be readily implemented using
on-board means.\@ LinkAhead therefore qualifies as a promising candidate for a scientific RDMS.\@

\subsection{Critical evaluation and outlook}\label{sec:linkahead:critical}

A common misunderstanding about LinkAhead is what it provides out of the box and what it can be used
for.  LinkAhead is not a tool to describe data objects following a specific ontology, but ontologies
can be implemented with LinkAhead in a straightforward manner, and it makes it easy to manage data
according to that ontology.  It is not an ELN either: ELNs focus on unintrusive interfaces for
manual data acquisition, but mostly leaving handling of data from other sources, or semantic data
searches, to other tools.  LinkAhead can be seen as a perfect complement to ELNs, its primary goal is
to make searching and linking of data beneficial for its users and to allow for automation of all
tasks.  One data source for this automation may be ELNs, but of course also other scientific data
acquisition appliances such as laboratory hardware, high-performance clusters or data repositories.

Similarly, LinkAhead does not enforce data to be FAIR.\@ However researchers can use LinkAhead to
implement a FAIR data management and to assure that they handle their data in a FAIR manner.  Data
transferred over the REST and GRPC interfaces use standardized formats such as XML for data
serialization, which can be understood by most programming interfaces.  Additionally, the internal
infrastructure of LinkAhead is being reworked to use UUIDs or other unique identifiers as primary keys
for all \mdm{Entities}.

As outlined in the previous section, LinkAhead fulfills most of the requirements and makes others
feasible for administrators and users.  This also implies that there is room for improvement, for
example by providing integrated connectors to ELNs or other data sources or templates for workflow
representations.

Along similar lines, LinkAhead is still lacking tools to seamlessly interchange data and data models
with RDF based systems.  In order to accelerate the general interoperability between data management
tools, LinkAhead has become part of the \emph{ELN consortium}\cite{_elnconsortium_2023}, an association
of interested parties with the aim to develop a common interchange format, based upon the
RO-Crate\cite{sefton_rocratemetadata_2023,soiland-reyes_packagingresearch_2022} specification.
While it is possible now already by external tools, full integration of existing vocabularies
represented in RDF serializations will further simplify FAIR data handling with LinkAhead.

When synchronizing data with LinkAhead, special attention has to be given to the relationship between
data from external sources (e.g., crawled files, ELNs) and records in the RDMS.\@ Different sources
can (usually by some error) have conflicting data, or entries in the RDMS can be changed manually by
users after their insertion. In our experience, this problem can not be solved in a general and
purely technical way. Instead, best practices have to be implemented as to where possible errors
should be corrected and whether some sources have precedence above each other. An RDMS like LinkAhead,
together with the crawler framework, can help administrators identify inconsistencies in the case of
two or more data sources. Through versioning, it is visible who and when maybe changed data
manually. How to optimize the help in recognizing potential conflicts, and in the end curate data
both in the RDMS and in the external sources, is subject of the authors' ongoing research.

Since LinkAhead does not receive institutional funding, the direction of its future development depends
on the actions of the community.  Therefore the immediate advancements will be shaped by the needs
of the current users of LinkAhead and of the company which currently provides commercial support for
it.  A current list of feature requests can be generated online.\cite{_linkaheadenhancement_2023} The
authors know of about a dozen institutions where LinkAhead is currently in use.  Together with the
growing user base we expect the software to persist for a significant amount of time.

LinkAhead may fall short in terms of performance against traditional SQL databases for very large
amounts of data.  To address this issue there is currently development underway to add a
virtualization layer which may use existing tabular data sources and present them in a configurable
way as native LinkAhead \mdm{Entities}.\cite{_linkaheadtrino_2023}

We are aware that the perceived ``usability'' is subject to personal preferences unless evaluated in
a controlled study.  We see the potential for a separate survey in the future which systematically
evaluates user experiences, workflows and the time and effort spent or gained by users of different
software approaches to a previously defined set of data management challenges.

\section{Conclusion}

We found that scientific research has specific needs to data management: Interoperability, agility,
adequate learning curves and early practical use.  Altogether we identified a set of eleven
requirements which we applied to multiple classes of technologies and tools and to LinkAhead, an agile
RDMS. Especially in the requirements cluster ``Semantic linkage, flexible data model, semantic
search'', previously existing tools show significant weaknesses, whereas LinkAhead offers a promising
outlook.

We hope that the open source license of LinkAhead will inspire more scientists to contribute to LinkAhead
and improve it in the areas of interoperability with existing standards.

\section{Appendix: Software}

\subsection{LinkAhead}\label{ssec:ref:linkahead}

The LinkAhead suite with the main libraries is published at Zenodo:\\
\url{https://zenodo.org/record/7752417}
(\href{https://doi.org/10.5281/zenodo.7752417}{DOI:10.5281/zenodo.7752417})

\subsection{List of LinkAhead libraries}\label{app:list}

The following libraries for programming client applications are publicly available:

\begin{description}
\item[Python] \url{https://gitlab.com/linkahead/linkahead-pylib} The Python client library can be used for
  third-party applications and is the foundation for several other libraries:
  \begin{description}
  \item[Advanced Python tools] \url{https://gitlab.com/linkahead/linkahead-advanced-user-tools} Additional
    high-level tools building upon the Python library, including a legacy implementation of the
    LinkAhead crawler.  These tools also include converters from JSON Schema to LinkAhead's data model.
  \item[Crawler] \url{https://gitlab.com/linkahead/linkahead-crawler} A new implementation of the LinkAhead
    crawler, also using the Python library.  Allows to validate data against a JSON Schema.
  \end{description}
\item[JavaScript] \url{https://gitlab.com/linkahead/linkahead-webui} The JavaScript library is part of the
  web user interface component.
\item[Protobuf API] \url{https://gitlab.com/linkahead/linkahead-proto} The gRPC API is defined via these
  Protobuf files.
\item[\Cpp{}] \url{https://gitlab.com/linkahead/linkahead-cpplib} The \Cpp{} library uses the gRPC API of
  LinkAhead.\@
\item[Octave] \url{https://gitlab.com/linkahead/linkahead-octavelib} The Octave/Matlab library is based
  upon the \Cpp{} library.
\item[Julia] \url{https://gitlab.com/linkahead/linkahead-julialib} The Julia library also is based upon
  the \Cpp{} library.
\end{description}

\section{Appendix: Query language comparison}\label{app:sparql}

As an example for nested queries in different query languages, we consider the search for female
UK-based writers in a certain time period, whose given or family name starts with the letter
``\texttt{M}''.  We used the RDF query language SPARQL with
Wikidata (\url{https://www.wikidata.org}) identifiers and LinkAhead's query language with
fictional but realistic identifier names.

The SPARQL query is as follows:

\lstset{language=SQL,morekeywords={PREFIX,java,rdf,rdfs,url,BIND,FILTER,SERVICE,WITH}}
\begin{lstlisting}
SELECT DISTINCT ?item ?itemLabel ?givenName ?familyName WHERE {
    ?item wdt:P31 wd:Q5;  # Any instance of a human.
          wdt:P27 wd:Q145; # citizenship in the United Kingdom
          wdt:P21 wd:Q6581072; # female
          wdt:P106 wd:Q36180; # writer
          wdt:P569 ?birthday;
          wdt:P570 ?diedon;
          wdt:P734 [rdfs:label ?familyName];
          wdt:P735 [rdfs:label ?givenName].
  FILTER(?birthday > "1870-01-01"^^xsd:dateTime
      && ?diedon < "1950-01-01"^^xsd:dateTime)
  FILTER(regex(?givenName, "M.*") || regex(?familyName, "M.*"))
     SERVICE wikibase:label { bd:serviceParam wikibase:language "en" }
  }
\end{lstlisting}

In contrast, the LinkAhead query looks like this:
\begin{lstlisting}
SELECT given_name, family_name FROM Writer
  WITH gender=f AND citizenship=UK AND birthday > 1870 AND death < 1950
       AND (given_name LIKE "M*" OR family_name LIKE "M*")
\end{lstlisting}

We understand that SPARQL and LinkAhead's query language have non-overlapping sets of features.  For
example, LinkAhead does not know about aliases for names, such as in multilingual environments.  On the
other hand, SPARQL has no native understanding of SI units and their conversion and it focuses on
experts instead of casual users.

%
%
% Acknowledgements, conflicts of interest, contributions & roles, and references
% will be set automatically at the end of the document
%
%

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-engine: luatex
%%% TeX-master: t
%%% End:

% LocalWords:  RDM RDMS lifecycle lifecycles foci RSpace eLabJournal Dataverse Invenio DSpace hoc
% LocalWords:  FSS llp mdm backend RML RO
