@conference {996, title = {Hidden Neuron Activation Analysis on Labeled Text Data}, booktitle = {K-CAP {\textquoteright}25: Knowledge Capture Conference 2025}, year = {2025}, month = {12/2025}, pages = {206 - 210}, publisher = {ACM}, organization = {ACM}, address = {USA}, abstract = {

Understanding the internal mechanisms of deep neural networks remains a central challenge in the field of Explainable Artificial Intelligence (XAI). With the rapid advancement of neural architectures in natural language processing (NLP), analyzing the role of hidden neurons in capturing and processing linguistic features has become increasingly important. This study investigates Hidden Neuron Activation Analysis on labeled text data to reveal how individual neurons contribute to a model{\textquoteright}s decision-making process. We propose a model-agnostic explainability framework for text classifiers that identifies concepts activating specific neurons involved in classification. An LSTM-based network is trained on the AG News topic classification dataset, comprising four distinct classes, and the final Dense layer with 64 neurons was analyzed. In addition, statistical analyses like the Mann-Whitney U Test is conducted to assess the robustness and reliability of the system. The statistical analysis shows that, concepts plays important role in the decision making process of neural network. Our findings enhance interpretability in NLP models and offer a foundation for optimizing neural architectures in text classification tasks.

}, keywords = {Concept-based Explanation, Dense Layer Analysis, Explainable AI, Hidden Neuron Analysis}, author = {Avishek Das and Abhilekha Dalal and Pascal Hitzler} } @conference {979, title = {Automating CNN Neuron Interpretation using Concept Induction}, booktitle = {THE 23RD INTERNATIONAL SEMANTIC WEB CONFERENCE, ISWC 2024}, year = {2024}, abstract = {

This paper presents an automation pipeline for interpreting hidden neuron activations in Convolutional Neural Networks (CNNs), a crucial objective of Explainable AI (XAI). Previously, our research group addressed this objective by employing concept induction and semantic reasoning using a concept hierarchy derived from the Wikipedia knowledge graph. However, the process was executed manually, taking several days to complete. In this study, we have fully automated the workflow, achieving consistent results while significantly reducing the execution time. The automation pipeline streamlines model training, data preparation, concept induction, image retrieval, classification, and statistical validation, thereby completely eliminating the manual intervention. This automation enables us to efficiently interpret and validate CNN neuron activations by modifying parameters, such as incorporating a broader range of training images and classes and examining additional concept induction results across various neuron layers using different analytical tools.

}, keywords = {Automation in AI, Deep Learning, Explainable Artificial Intelligence, Knowledge Graph, Semantic Web}, author = {Samatha Ereshi Akkamahadevi and Abhilekha Dalal and Pascal Hitzler} } @proceedings {973, title = {Error-margin Analysis for Hidden Neuron Activation Labels}, journal = {18th International Conference on Neural-Symbolic Learning and Reasoning, NeSy 2024}, year = {2024}, publisher = {Springer }, abstract = {

Understanding how high-level concepts are represented with- in artificial neural networks is a fundamental challenge in the field of arti- ficial intelligence. While existing literature in explainable AI emphasizes the importance of labeling neurons with concepts to understand their functioning, they mostly focus on identifying what stimulus activates a neuron in most cases; this corresponds to the notion of recall in informa- tion retrieval. We argue that this is only the first-part of a two-part job; it is imperative to also investigate neuron responses to other stimuli, i.e., their precision. We call this the neuron label{\textquoteright}s error margin.

}, keywords = {CNN, Concept Induction, Explainable AI}, author = {Abhilekha Dalal and Rushrukh Rayan and Pascal Hitzler} } @conference {972, title = {On the Value of Labeled Data and Symbolic Methods for Hidden Neuron Activation Analysis}, booktitle = {18th International Conference on Neural-Symbolic Learning and Reasoning, NeSy 2024}, year = {2024}, publisher = {Springer }, organization = {Springer }, abstract = {

We introduce a novel model-agnostic post-hoc Explainable AI method that provides meaningful interpretations for hidden neuron activations in a Convolutional Neural Network. Our approach uses a Wikipedia-derived concept hierarchy with approx. 2 million classes as background knowledge, and deductive reasoning based Concept Induc- tion for explanation generation. Additionally, we explore and compare the capabilities of off-the-shelf pre-trained multimodal-based explainable methods. Our evaluation shows that our neurosymbolic method holds a competitive edge in both quantitative and qualitative aspects.

}, keywords = {CNN, Concept Induction, Explainable AI, LLM}, author = {Abhilekha Dalal and Rushrukh Rayan and Adrita Barua and Eugene Y. Vasserman and Kamruzzaman Sarker and Pascal Hitzler} }