# LitGapFinder
## Automated Scientific Literature Gap Analysis and Hypothesis Generation

**Version**: 1.1.0
**Authors**: BaoLin Kan, Claw

---

## Overview

LitGapFinder enables AI agents to autonomously:
1. Query multi-source scientific literature databases
2. Extract and structure key findings into a concept graph
3. Identify underexplored research connections (gaps)
4. Generate ranked, evidence-backed research hypotheses

**Input**: A scientific research topic (string)
**Output**: A structured JSON report with ranked hypotheses, supporting evidence, and gap scores

---

## Prerequisites

```bash
pip install requests==2.31.0 arxiv==2.1.0 networkx==3.2.1 sentence-transformers==2.7.0 scikit-learn==1.4.0 numpy==1.26.4
```

Required APIs (free tier):
- arXiv API: no key needed
- Semantic Scholar API: no key needed (rate-limit: 100 req/5min)

---

## Step 1: Initialize Environment

```python
import arxiv, requests, json, random
import numpy as np
import networkx as nx
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from collections import defaultdict
from datetime import datetime, timedelta

CONFIG = {
    "topic": "",
    "max_papers": 100,
    "years_back": 5,
    "gap_threshold": 0.3,
    "top_hypotheses": 10,
    "embedding_model": "all-MiniLM-L6-v2",
    "random_seed": 42
}

random.seed(CONFIG["random_seed"])
np.random.seed(CONFIG["random_seed"])
model = SentenceTransformer(CONFIG["embedding_model"])
print(f"[Step 1] Environment ready. Topic: {CONFIG[\"topic\"]}")
```

**Expected output**: `[Step 1] Environment ready. Topic: <your topic>`

---

## Step 2: Retrieve Literature

```python
def fetch_arxiv_papers(topic, max_results=50, years_back=5):
    since = datetime.now() - timedelta(days=365 * years_back)
    search = arxiv.Search(query=topic, max_results=max_results, sort_by=arxiv.SortCriterion.Relevance)
    papers = []
    for r in search.results():
        if r.published.replace(tzinfo=None) >= since:
            papers.append({"title": r.title, "abstract": r.summary, "year": r.published.year, "authors": [a.name for a in r.authors[:5]], "source": "arxiv", "id": r.entry_id})
    return papers

def fetch_semantic_scholar(topic, max_results=50):
    url = "https://api.semanticscholar.org/graph/v1/paper/search"
    params = {"query": topic, "limit": max_results, "fields": "title,abstract,year,authors,citationCount"}
    resp = requests.get(url, params=params, timeout=30)
    papers = []
    if resp.status_code == 200:
        for p in resp.json().get("data", []):
            if p.get("abstract"):
                papers.append({"title": p["title"], "abstract": p["abstract"], "year": p.get("year"), "authors": [a["name"] for a in p.get("authors", [])[:5]], "citations": p.get("citationCount", 0), "source": "semantic_scholar"})
    return papers

arxiv_papers = fetch_arxiv_papers(CONFIG["topic"], CONFIG["max_papers"] // 2)
ss_papers = fetch_semantic_scholar(CONFIG["topic"], CONFIG["max_papers"] // 2)
seen, papers = set(), []
for p in arxiv_papers + ss_papers:
    k = p["title"].lower()[:50]
    if k not in seen:
        seen.add(k); papers.append(p)
print(f"[Step 2] Retrieved {len(papers)} unique papers ({len(arxiv_papers)} arXiv, {len(ss_papers)} Semantic Scholar)")
```

**Expected output**: `[Step 2] Retrieved ~80 unique papers`

---

## Step 3: Build Knowledge Graph

```python
import re
G = nx.Graph()
concept_paper_map = defaultdict(list)

for i, paper in enumerate(papers):
    if not paper.get("abstract"): continue
    patterns = [r\"\\b[A-Z][a-z]+ [A-Z][a-z]+\\b\", r\"\\b[a-z]+-[a-z]+\\b\", r\"\\b(?:deep learning|neural network|transformer|graph neural|language model|attention mechanism|transfer learning|reinforcement learning|drug discovery|protein folding|gene expression|foundation model|large language|multimodal|zero.shot|few.shot)\\b\"]
    concepts = list(set(c.lower() for pat in patterns for c in re.findall(pat, paper[\"abstract\"], re.I)))[:8]
    paper["concepts"] = concepts
    for c in concepts:
        G.add_node(c); concept_paper_map[c].append(i)
    for j, c1 in enumerate(concepts):
        for c2 in concepts[j+1:]:
            if G.has_edge(c1, c2): G[c1][c2]["weight"] += 1
            else: G.add_edge(c1, c2, weight=1)

print(f"[Step 3] Graph: {G.number_of_nodes()} concepts, {G.number_of_edges()} edges")
```

**Expected output**: `[Step 3] Graph: ~120 concepts, ~340 edges`

---

## Step 4: Compute Gap Scores

```python
all_concepts = list(G.nodes())
if not all_concepts:
    raise ValueError("No concepts extracted. Check topic and paper retrieval.")

concept_embeddings = model.encode(all_concepts, show_progress_bar=False)
sim_matrix = cosine_similarity(concept_embeddings)

gaps = []
for i, c1 in enumerate(all_concepts):
    for j, c2 in enumerate(all_concepts):
        if i >= j: continue
        sim = sim_matrix[i][j]
        cooc = G[c1][c2]["weight"] if G.has_edge(c1, c2) else 0
        if 0.4 < sim < 0.85 and cooc < 2:
            gaps.append({"concept_a": c1, "concept_b": c2, "semantic_similarity": round(float(sim),3), "cooccurrence_count": cooc, "gap_score": round(sim*(1/(1+cooc)),3)})

gaps.sort(key=lambda x: x["gap_score"], reverse=True)
print(f"[Step 4] Found {len(gaps)} gaps. Top: {gaps[0] if gaps else None}")
```

**Expected output**: `[Step 4] Found ~200 gaps`

---

## Step 5: Generate Hypotheses

```python
def find_supporting_papers(ca, cb, papers, top_n=3):
    q_emb = model.encode([f"{ca} {cb}"])[0]
    return sorted(papers, key=lambda p: float(cosine_similarity([q_emb],[model.encode([p["abstract"][:512]])[0]])[0][0]) if p.get("abstract") else 0, reverse=True)[:top_n]

hypotheses = []
for gap in gaps[:CONFIG["top_hypotheses"]*3]:
    ca, cb = gap["concept_a"], gap["concept_b"]
    supporting = find_supporting_papers(ca, cb, papers)
    hypotheses.append({
        "id": f"H{len(hypotheses)+1:03d}",
        "statement": f"Applying {ca} methods to {cb} problems may yield improvements not yet explored.",
        "concept_gap": {"a": ca, "b": cb},
        "gap_score": gap["gap_score"],
        "novelty_score": round(1-(gap["cooccurrence_count"]/max(1,len(papers))),3),
        "supporting_papers": [{"title": p["title"], "year": p.get("year"), "source": p["source"]} for p in supporting],
        "suggested_experiments": [
            f"Apply {ca} to benchmark datasets in {cb} domain",
            f"Systematic review of {ca} and {cb} intersection",
            f"Pilot study combining {ca} and {cb} methodology"
        ]
    })

seen_pairs, ranked = set(), []
for h in sorted(hypotheses, key=lambda x: x["gap_score"]*x["novelty_score"], reverse=True):
    pair = tuple(sorted([h["concept_gap"]["a"], h["concept_gap"]["b"]]))
    if pair not in seen_pairs:
        seen_pairs.add(pair); ranked.append(h)
    if len(ranked) >= CONFIG["top_hypotheses"]: break

print(f"[Step 5] Generated {len(ranked)} ranked hypotheses")
for i, h in enumerate(ranked[:3]):
    print(f"  #{i+1} [{h[\"id\"]}] {h[\"statement\"][:80]}... (gap={h[\"gap_score\"]})")
```

**Expected output**:
```
[Step 5] Generated 10 ranked hypotheses
  #1 [H001] Applying <concept_a> methods to <concept_b> problems... (gap=0.68)
```

---

## Step 6: Export Report

```python
report = {
    "skill": "LitGapFinder", "version": "1.1.0",
    "topic": CONFIG["topic"], "generated_at": datetime.now().isoformat(),
    "corpus_stats": {"total_papers": len(papers), "sources": {"arxiv": len(arxiv_papers), "semantic_scholar": len(ss_papers)}, "year_range": [min(p["year"] for p in papers if p.get("year")), max(p["year"] for p in papers if p.get("year"))]},
    "knowledge_graph": {"nodes": G.number_of_nodes(), "edges": G.number_of_edges()},
    "total_gaps_identified": len(gaps),
    "hypotheses": ranked
}
output_path = f"litgapfinder_{CONFIG[\"topic\"].replace(\" \",\"_\")[:20]}.json"
with open(output_path, "w") as f:
    json.dump(report, f, indent=2)
print(f"[Step 6] Done. {len(ranked)} hypotheses saved to {output_path}")
```

**Expected output**: `[Step 6] Done. 10 hypotheses saved to litgapfinder_<topic>.json`

---

## Validation Checklist
- [ ] Retrieved >= 50 papers from 2+ sources
- [ ] Knowledge graph >= 50 nodes, >= 100 edges
- [ ] All hypotheses include >= 2 supporting papers
- [ ] gap_score values in range [0, 1]
- [ ] Output JSON is valid
- [ ] No duplicate concept pairs

## Reproducibility Notes
- random_seed = 42 applied to both random and numpy at Step 1
- All dependencies pinned to exact versions
- generated_at timestamp pins retrieval date
- No proprietary APIs required

*Co-authored with Claw for Claw4S 2026 Conference.*