@inbook {994, title = {Neurosymbolic Hidden Neuron Analysis in Convolutional Neural Networks}, booktitle = {Neuro-Symbolic AI: Bridging the Gap Between Neural Networks and Symbolic Reasoning}, year = {2026}, publisher = {Elsevier}, organization = {Elsevier}, chapter = {8}, abstract = {

This tutorial introduces a step-by-step, deductive pipeline for making the inner workings of neural networks more transparent by assigning human-understandable concepts to hidden neuron activations. The approach automatically maps neuron behavior to symbolic concepts drawn from structured knowledge sources and attaches an error margin to each label, providing a measure of confidence in its precision. While demonstrated in detail on the ADE20k scene dataset---including single-concept neurons, multiple neurons contributing to the same concept, and multi-concept neurons---the method is also applied to the SUN2012 dataset and adapted for a text classification task, highlighting its generalizability across modalities. The chapter is designed to be practical and educational, focusing on a replicable methodology that readers can adapt to varied applications. Through worked examples, visualizations, and evaluation strategies, the tutorial offers a clear, reusable framework for concept-based neuron analysis in both vision and language models.

}, author = {Abhilekha Dalal and Moumita Sen Sarma and Avishek Das and Samatha E. Akkamahadevi and Eugene Y. Vasserman and Pascal Hitzler} } @conference {996, title = {Hidden Neuron Activation Analysis on Labeled Text Data}, booktitle = {K-CAP {\textquoteright}25: Knowledge Capture Conference 2025}, year = {2025}, month = {12/2025}, pages = {206 - 210}, publisher = {ACM}, organization = {ACM}, address = {USA}, abstract = {

Understanding the internal mechanisms of deep neural networks remains a central challenge in the field of Explainable Artificial Intelligence (XAI). With the rapid advancement of neural architectures in natural language processing (NLP), analyzing the role of hidden neurons in capturing and processing linguistic features has become increasingly important. This study investigates Hidden Neuron Activation Analysis on labeled text data to reveal how individual neurons contribute to a model{\textquoteright}s decision-making process. We propose a model-agnostic explainability framework for text classifiers that identifies concepts activating specific neurons involved in classification. An LSTM-based network is trained on the AG News topic classification dataset, comprising four distinct classes, and the final Dense layer with 64 neurons was analyzed. In addition, statistical analyses like the Mann-Whitney U Test is conducted to assess the robustness and reliability of the system. The statistical analysis shows that, concepts plays important role in the decision making process of neural network. Our findings enhance interpretability in NLP models and offer a foundation for optimizing neural architectures in text classification tasks.

}, keywords = {Concept-based Explanation, Dense Layer Analysis, Explainable AI, Hidden Neuron Analysis}, author = {Avishek Das and Abhilekha Dalal and Pascal Hitzler} } @conference {979, title = {Automating CNN Neuron Interpretation using Concept Induction}, booktitle = {THE 23RD INTERNATIONAL SEMANTIC WEB CONFERENCE, ISWC 2024}, year = {2024}, abstract = {

This paper presents an automation pipeline for interpreting hidden neuron activations in Convolutional Neural Networks (CNNs), a crucial objective of Explainable AI (XAI). Previously, our research group addressed this objective by employing concept induction and semantic reasoning using a concept hierarchy derived from the Wikipedia knowledge graph. However, the process was executed manually, taking several days to complete. In this study, we have fully automated the workflow, achieving consistent results while significantly reducing the execution time. The automation pipeline streamlines model training, data preparation, concept induction, image retrieval, classification, and statistical validation, thereby completely eliminating the manual intervention. This automation enables us to efficiently interpret and validate CNN neuron activations by modifying parameters, such as incorporating a broader range of training images and classes and examining additional concept induction results across various neuron layers using different analytical tools.

}, keywords = {Automation in AI, Deep Learning, Explainable Artificial Intelligence, Knowledge Graph, Semantic Web}, author = {Samatha Ereshi Akkamahadevi and Abhilekha Dalal and Pascal Hitzler} } @proceedings {973, title = {Error-margin Analysis for Hidden Neuron Activation Labels}, journal = {18th International Conference on Neural-Symbolic Learning and Reasoning, NeSy 2024}, year = {2024}, publisher = {Springer }, abstract = {

Understanding how high-level concepts are represented with- in artificial neural networks is a fundamental challenge in the field of arti- ficial intelligence. While existing literature in explainable AI emphasizes the importance of labeling neurons with concepts to understand their functioning, they mostly focus on identifying what stimulus activates a neuron in most cases; this corresponds to the notion of recall in informa- tion retrieval. We argue that this is only the first-part of a two-part job; it is imperative to also investigate neuron responses to other stimuli, i.e., their precision. We call this the neuron label{\textquoteright}s error margin.

}, keywords = {CNN, Concept Induction, Explainable AI}, author = {Abhilekha Dalal and Rushrukh Rayan and Pascal Hitzler} } @mastersthesis {993, title = {Understanding Hidden Neuron Activations Using Structured Background Knowledge and Deductive Reasoning}, volume = {PhD}, year = {2024}, abstract = {

A central challenge in Explainable AI (XAI) is accurately interpreting hidden neuron activations in deep neural networks (DNNs). Accurate interpretations help demystify the black-box nature of deep learning models by explaining what the system internally detects as relevant in the input. While some existing methods show that hidden neuron activations can be human-interpretable, systematic and automated approaches leveraging background knowledge remain underexplored. This thesis introduces a novel model-agnostic post-hoc XAI method that integrates a Wikipedia-derived concept hierarchy of approximately 2 million classes as background knowledge and employs OWL-reasoning-based Concept Induction to generate explanations. Our approach automatically assigns meaningful class expressions to neurons in the dense layers of Convolutional Neural Networks, outperforming prior methods both quantitatively and qualitatively.

In addition, we argue that understanding neuron behavior requires not only identifying what activates a neuron (recall) but also examining its precision{\textemdash}how it responds to other stimuli, which we define as the neuron{\textquoteright}s error margin, enhancing the granularity of neuron interpretation.

To visualize these findings, we present ConceptLens, an innovative tool that visualizes neuron activations and error margins. ConceptLens offers insights into the confidence levels of neuron activations and enables an intuitive understanding of neuron behavior through visual bar charts. Together, these contributions offer a holistic approach to interpreting DNNs, advancing the explainability and transparency of AI models.

}, author = {Abhilekha Dalal} } @conference {972, title = {On the Value of Labeled Data and Symbolic Methods for Hidden Neuron Activation Analysis}, booktitle = {18th International Conference on Neural-Symbolic Learning and Reasoning, NeSy 2024}, year = {2024}, publisher = {Springer }, organization = {Springer }, abstract = {

We introduce a novel model-agnostic post-hoc Explainable AI method that provides meaningful interpretations for hidden neuron activations in a Convolutional Neural Network. Our approach uses a Wikipedia-derived concept hierarchy with approx. 2 million classes as background knowledge, and deductive reasoning based Concept Induc- tion for explanation generation. Additionally, we explore and compare the capabilities of off-the-shelf pre-trained multimodal-based explainable methods. Our evaluation shows that our neurosymbolic method holds a competitive edge in both quantitative and qualitative aspects.

}, keywords = {CNN, Concept Induction, Explainable AI, LLM}, author = {Abhilekha Dalal and Rushrukh Rayan and Adrita Barua and Eugene Y. Vasserman and Kamruzzaman Sarker and Pascal Hitzler} } @article {913, title = {Bridging RDF and Property Graphs: Linking KnowWhereGraph and SPOKE}, year = {2023}, author = {Shirly Stephen and Cogan Shimizu and Pascal Hitzler and Karthik Soman and Peter W Rose and John H Morris and Sergio E Baranzini and Krzysztof Janowicz and Antrea Christou and Abhilekha Dalal and Kitty Currier and Mark Schildhauer} } @article {905, title = {Explaining Deep Learning Hidden Neuron Activations using Concept Induction}, year = {2023}, abstract = {

One of the current key challenges in Explainable AI is in correctly interpreting activations of hidden neurons. It seems evident that accurate interpretations thereof would provide insights into the question what a deep learning system has internally detected as relevant on the input, thus lifting some of the black box character of deep learning systems.

The state of the art on this front indicates that hidden node activations appear to be interpretable in a way that makes sense to humans, at least in some cases. Yet, systematic automated methods that would be able to first hypothesize an interpretation of hidden neuron activations, and then verify it, are mostly missing.\ 

In this paper, we provide such a method and demonstrate that it provides meaningful interpretations. It is based on using large-scale background knowledge -- a class hierarchy of approx. 2 million classes curated from the Wikipedia Concept Hierarchy -- together with a symbolic reasoning approach called concept induction\ based on description logics that was originally developed for applications in the Semantic Web field.\ 

Our results show that we can automatically attach meaningful labels from the background knowledge to individual neurons in the dense layer of a Convolutional Neural Network through a hypothesis and verification process.

}, author = {Abhilekha Dalal and Md Kamruzzaman Sarker and Adrita Barua and Pascal Hitzler} } @article {906, title = {The KnowWhereGraph Ontology}, year = {2023}, author = {Cogan Shimizu and Shirly Stephen and Kitty Currier and Pascal Hitzler and Rui Zhu and Krzysztof Janowicz and Mark Schildhauer and Mohammad Saeid Mahdavinejad and Abhilekha Dalal and Adrita Barua and Ling Cai and Gengchen Mai and Zhangyu Wang and Yuanyuan Tian and Sanaz Saki Norouzi and Zilong Liu and Meilin Shi and Colby K. Fisher} } @conference {916, title = {The KnowWhereGraph Ontology: A Showcase}, year = {2023}, author = {Cogan Shimizu and Shirly Stephen and Rui Zhu and Kitty Currier and Mark Schildhauer and Dean Rehberger and Pascal Hitzler and Krzysztof Janowicz and Colby K. Fisher and Mohammad Saeid Mahdavinejad and Antrea Christou and Adrita Barua and Abhilekha Dalal and Sanaz Saki Norouzi and Zilong Liu and Meilin Shi and Ling Cai and Gengchen Mai and Zhangyu Wang and Yuanyuan Tian} } @conference {914, title = {KnowWhereGraph-Lite: A Perspective of the KnowWhereGraph}, booktitle = {KGSWC 2023}, year = {2023}, author = {Cogan Shimizu and Shirly Stephen and Antrea Christou and Kitty Currier and Mohammad Saeid Mahdavinejad and Sanaz Saki Norouzi and Abhilekha Dalal and Adrita Barua and Colby K. Fisher and Anthony D{\textquoteright}Onofrio and Thomas Thelen and Krzysztof Janowicz and Dean Rehberger and Mark Schildhauer and Pascal Hitzler} } @article {915, title = {Openness and Transparency in Academic Publishing: A Decade of Data from the Semantic Web Journal}, year = {2023}, author = {Pascal Hitzler and Krzysztof Janowicz and Cogan Shimizu and Abhilekha Dalal and Aaron Eberhart and Andrew Eells and Sanaz Saki Norouzi} } @conference {867, title = {Bridging Upper Ontology and Modular Ontology Modeling: A Tool and Evaluation}, booktitle = {KGSWC-2021}, year = {2021}, abstract = {

Ontologies are increasingly used as schema for knowledge graphs in many application areas. As such, there are a variety of different approaches for their development. In this paper, we describe and evaluate UAO (for Upper Ontology Alignment Tool), which is an extension to CoModIDE, a graphical prote{\textquoteright}ge{\textquoteright}\ plugin for modular ontology modeling. UAO enables ontology engineers to combine modular ontology modeling with a more traditional ontology modeling approach based on upper ontologies. We posit -- and our evaluation supports this claim -- that the tool does indeed makes it easier to combine both approaches. Thus, UAO enables a best-of-both-worlds approach. The evaluation consists of a user study, and the results show that performing typical manual alignment modeling tasks is relatively easier with UAO than doing it with porte{\textquoteright}ge{\textquoteright} alone, in terms of the time required to complete the task and improving the correctness of the output. Additionally, our test subjects provided significantly higher ratings on the System Utilization Scale for UOA.

}, author = {Abhilekha Dalal and Cogan Shimizu and Pascal Hitzler} } @conference {816, title = {Modular Ontology Modeling Meets Upper Ontologies: The Upper Ontology Alignment Tool}, booktitle = {The 19th International Semantic Web Conference}, volume = {2721}, year = {2020}, month = {10/2020}, pages = {119-124}, abstract = {

We provide an extension to the Prote{\textquoteright}ge{\textquoteright}-based modular\ ontology engineering tool CoModIDE, in order to make it possible for ontology engineers to adhere to traditional ontology modeling processes based on upper or foundational ontologies. As a bridge between the more recently proposed modular ontology modeling approach and more classical ones based on foundational ontologies, it enables a best-of-both-worlds approach for ontology engineering.

}, author = {Abhilekha Dalal and Cogan Shimizu and Pascal Hitzler} } @mastersthesis {859, title = {Modular Ontology Modeling Meets Upper Ontologies: The Upper Ontology Alignment Tool}, volume = {Masters}, year = {2020}, month = {11/2020}, pages = {35}, school = {Kansas State University}, address = {Manhattan}, abstract = {

Ontology modeling has become a primary approach to schema generation for data integration and knowledge graphs in many application areas. The quest for efficient approaches to model useful and re-useable ontologies has led to different ontology creation proposals over the years. The project focuses on two major approaches, modeling using a top-level ontology, and the other is modular ontology modeling.

The traditional approach is based on top-level ontology, and the strategy is to utilize ontology that is comprehensive enough to cover a broad spectrum of domains through their universal terminologies. In this way, all domain ontologies share a common top-level formal ontology in which their respective root nodes can be defined, and hence consistency is assured across the knowledge graph. Nevertheless, the most recent approach is quite different and is a refinement of the eXtreme Ontology Design methodology based on the ontology design patterns. Whole ontology is viewed as a collection of interconnected modules, and modules are developed around the classified fundamental notions according to experts{\textquoteright} terminology or the use-case. Having developed modules in a fashion of divide and conquer, these modules are shareable and reusable among some other ontology if needed, and consequently, the ontology being FAIR is justified (findable, accessible, interoperable, and reusable).

Although, it has been argued that there are advantages to either paradigm, it is possible to have a combination of both approaches mentioned earlier, depending upon the use-case or the preferences of the ontology engineers. We provide an extension to the Prot{\'e}g{\'e} - based modular ontology engineering tool CoModIDE, in order to make it possible for ontology engineers to follow traditional, ad-hoc ontology modeling approach, alongside more modern paradigms such as modular ontology engineering. The project focuses on domain-level ontology developers or organizations dealing with ontology development, which may get help through the plugin in minimizing the tooling gap to unite paradigms and develop robust, flexible ontologies suitable to their needs. As a bridge between the more recently proposed modular ontology modeling approach and more classical ones based on foundational ontologies, it enables a best-of-both-worlds approach for ontology engineering.\ 

}, author = {Abhilekha Dalal} }