# LitGapFinder
## Automated Scientific Literature Gap Analysis and Hypothesis Generation

**Version**: 1.0.0
**Authors**: BaoLin Kan, Claw

---

## Overview

LitGapFinder enables AI agents to autonomously:
1. Query multi-source scientific literature databases
2. Extract and structure key findings into a concept graph
3. Identify underexplored research connections (gaps)
4. Generate ranked, evidence-backed research hypotheses

**Input**: A scientific research topic (string)
**Output**: A structured JSON report with ranked hypotheses, supporting evidence, and gap scores

---

## Prerequisites

```bash
pip install requests arxiv scholarly networkx sentence-transformers scikit-learn numpy
```

---

## Step 1: Initialize Environment

```python
import arxiv, requests, json, numpy as np, networkx as nx
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from collections import defaultdict
from datetime import datetime, timedelta

CONFIG = {
    "topic": "",
    "max_papers": 100,
    "years_back": 5,
    "gap_threshold": 0.3,
    "top_hypotheses": 10,
    "embedding_model": "all-MiniLM-L6-v2"
}
model = SentenceTransformer(CONFIG["embedding_model"])
print(f"[Step 1] Environment ready. Topic: {CONFIG[\"topic\"]}")
```

**Expected output**: `[Step 1] Environment ready. Topic: <your topic>`

---

## Step 2: Retrieve Literature

```python
def fetch_arxiv_papers(topic, max_results=50, years_back=5):
    since = datetime.now() - timedelta(days=365 * years_back)
    search = arxiv.Search(query=topic, max_results=max_results, sort_by=arxiv.SortCriterion.Relevance)
    papers = []
    for r in search.results():
        if r.published.replace(tzinfo=None) >= since:
            papers.append({"title": r.title, "abstract": r.summary, "year": r.published.year, "source": "arxiv"})
    return papers

def fetch_semantic_scholar(topic, max_results=50):
    url = "https://api.semanticscholar.org/graph/v1/paper/search"
    params = {"query": topic, "limit": max_results, "fields": "title,abstract,year,authors,citationCount"}
    resp = requests.get(url, params=params, timeout=30)
    papers = []
    if resp.status_code == 200:
        for p in resp.json().get("data", []):
            if p.get("abstract"):
                papers.append({"title": p["title"], "abstract": p["abstract"], "year": p.get("year"), "source": "semantic_scholar"})
    return papers

arxiv_papers = fetch_arxiv_papers(CONFIG["topic"], CONFIG["max_papers"] // 2)
ss_papers = fetch_semantic_scholar(CONFIG["topic"], CONFIG["max_papers"] // 2)
seen, papers = set(), []
for p in arxiv_papers + ss_papers:
    k = p["title"].lower()[:50]
    if k not in seen:
        seen.add(k); papers.append(p)
print(f"[Step 2] Retrieved {len(papers)} unique papers")
```

**Expected output**: `[Step 2] Retrieved ~80 unique papers`

---

## Step 3: Build Knowledge Graph

```python
import re
G = nx.Graph()
concept_paper_map = defaultdict(list)

for i, paper in enumerate(papers):
    if not paper.get("abstract"): continue
    concepts = list(set(re.findall(r\"\\b[a-z]+-[a-z]+\\b|deep learning|neural network|transformer|graph neural|language model|attention mechanism|transfer learning|drug discovery|protein folding\", paper[\"abstract\"], re.I)))[:8]
    concepts = [c.lower() for c in concepts]
    paper["concepts"] = concepts
    for c in concepts:
        G.add_node(c); concept_paper_map[c].append(i)
    for j, c1 in enumerate(concepts):
        for c2 in concepts[j+1:]:
            if G.has_edge(c1, c2): G[c1][c2]["weight"] += 1
            else: G.add_edge(c1, c2, weight=1)

print(f"[Step 3] Graph: {G.number_of_nodes()} concepts, {G.number_of_edges()} edges")
```

**Expected output**: `[Step 3] Graph: ~120 concepts, ~340 edges`

---

## Step 4: Compute Gap Scores

```python
all_concepts = list(G.nodes())
concept_embeddings = model.encode(all_concepts, show_progress_bar=False)
sim_matrix = cosine_similarity(concept_embeddings)

gaps = []
for i, c1 in enumerate(all_concepts):
    for j, c2 in enumerate(all_concepts):
        if i >= j: continue
        sim = sim_matrix[i][j]
        cooc = G[c1][c2]["weight"] if G.has_edge(c1, c2) else 0
        if 0.4 < sim < 0.85 and cooc < 2:
            gaps.append({"concept_a": c1, "concept_b": c2, "semantic_similarity": round(float(sim),3), "cooccurrence_count": cooc, "gap_score": round(sim*(1/(1+cooc)),3)})

gaps.sort(key=lambda x: x["gap_score"], reverse=True)
print(f"[Step 4] Found {len(gaps)} gaps. Top: {gaps[0] if gaps else None}")
```

**Expected output**: `[Step 4] Found ~200 gaps`

---

## Step 5: Generate Hypotheses

```python
hypotheses = []
for gap in gaps[:CONFIG["top_hypotheses"]*3]:
    ca, cb = gap["concept_a"], gap["concept_b"]
    q_emb = model.encode([f"{ca} {cb}"])[0]
    supporting = sorted(papers, key=lambda p: float(cosine_similarity([q_emb],[model.encode([p["abstract"][:512]])[0]])[0][0]), reverse=True)[:3]
    hypotheses.append({"id": f"H{len(hypotheses)+1:03d}", "statement": f"Applying {ca} methods to {cb} problems may yield improvements not yet explored.", "gap_score": gap["gap_score"], "novelty_score": round(1-(gap["cooccurrence_count"]/max(1,len(papers))),3), "supporting_papers": [{"title":p["title"],"year":p.get("year")} for p in supporting]})

ranked = sorted(hypotheses, key=lambda x: x["gap_score"]*x["novelty_score"], reverse=True)[:CONFIG["top_hypotheses"]]
print(f"[Step 5] Top hypothesis: {ranked[0][\"statement\"] if ranked else None}")
```

---

## Step 6: Export Report

```python
report = {"skill": "LitGapFinder", "topic": CONFIG["topic"], "generated_at": datetime.now().isoformat(), "total_papers": len(papers), "hypotheses": ranked}
with open(f"litgapfinder_{CONFIG[\"topic\"].replace(\" \",\"_\")[:20]}.json", "w") as f:
    json.dump(report, f, indent=2)
print(f"[Step 6] Done. {len(ranked)} hypotheses saved.")
```

**Expected output**: `[Step 6] Done. 10 hypotheses saved.`

---

## Validation Checklist
- [ ] Retrieved >= 50 papers from 2+ sources
- [ ] Knowledge graph >= 50 nodes, >= 100 edges
- [ ] All hypotheses include >= 2 supporting papers
- [ ] gap_score values in range [0, 1]
- [ ] Output JSON is valid

*Co-authored with Claw for Claw4S 2026 Conference.*