Initial version

sok/management/commands/citations.py

@@ -0,0 +1,60 @@
+from typing import Set, Tuple
+
+from django.core.management.base import BaseCommand, CommandParser
+from django.db.models import Count, Q
+
+from sok.models import Publication
+
+
+class Command(BaseCommand):
+
+	def echo(self, msg: str):
+		self.stdout.write(msg)
+
+	# BaseCommand
+
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('--min-citations', type=int, default=0)
+		parser.add_argument('--pk', action='store_true')
+
+	def graphviz(self, pk: bool, min_citations: int) -> None:
+		publications = Publication.objects.filter(exclusion_criteria__isnull=True).annotate(
+			citation_count=Count(
+				'referenced_by',
+				filter=Q(exclusion_criteria__isnull=True),
+				distinct=True,
+			),
+		).filter(citation_count__gte=min_citations)
+
+		self.echo("digraph G {")
+		self.echo("\trankdir = RL;")
+
+		graph: Set[Tuple[int, int]] = set()
+
+		for publication in publications:
+
+			if publication.stage != 'primary':
+				continue
+
+			for reference in publication.relevant_references:
+
+				if reference not in publications:
+					continue
+
+				graph.add((publication.pk, reference.pk))
+				if (reference.pk, publication.pk) in graph:
+					self.stderr.write(self.style.ERROR(
+						f"CYCLE: {publication.cite_key} <-> {reference.cite_key}"
+					))
+
+				if pk:
+					self.echo(f'\t"{publication.pk}" -> "{reference.pk}";')
+				else:
+					self.echo(f'\t"{publication.cite_key}" -> "{reference.cite_key}";')
+
+		self.echo("}")
+
+	def handle(self, *args, **options) -> None:
+		min_citations: int = options['min_citations']
+		pk: bool = options['pk']
+		self.graphviz(pk, min_citations)

sok/management/commands/dblpimport.py

@@ -0,0 +1,497 @@
+import html
+import io
+import pickle
+import string
+import xml.sax
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from urllib.parse import urlparse
+
+import requests
+
+from django.db import transaction
+from django.core.management.base import BaseCommand, CommandParser, CommandError
+
+from sok.models import (
+	Author,
+	Publication,
+	PublicationAuthor,
+	PublicationSource,
+	SearchTerm,
+	Source,
+)
+
+
+Attributes = xml.sax.xmlreader.AttributesImpl
+
+
+PUBLICATIONS = {
+	'article',
+	'inproceedings',
+	'proceedings',
+	'book',
+	'incollection',
+	'phdthesis',
+	'mastersthesis',
+	'www',
+	'person',
+	'data',
+}
+
+CITE_KEY_PREFIX = 'DBLP:'
+DUMP_PATH = Path('dblp') / 'dblp-2021-03-01.xml'
+
+
+def strip_cite_key_prefix(value: str) -> str:
+	if value.startswith(CITE_KEY_PREFIX):
+		return value[len(CITE_KEY_PREFIX):]
+	return value
+
+
+def strip_issue_from_page(value: str) -> int:
+	return int(''.join(c for c in value.split(':')[-1] if c in string.digits))
+
+
+def clean_title(value: str) -> str:
+	if value.endswith('.'):
+		return value[:-1]
+	return value
+
+
+def parse_pages(raw: str) -> Tuple[int, int]:
+	# Observed:
+	# - '1-10'
+	# - '1'
+	# - '16:1-16:10'
+	# - 'I-X, 1-66'
+	# - '186-'
+
+	pages = raw.split(', ')[-1].split('-')
+
+	if 2 == len(pages):
+		first, last = pages
+		if last != '':
+			return (strip_issue_from_page(first), strip_issue_from_page(last))
+		else:
+			pages = [first]
+
+	if 1 == len(pages):
+		page = strip_issue_from_page(pages[0])
+		return (page, page)
+
+	raise NotImplementedError(f"Unexpected value for <pages>: {raw}")
+
+
+class FinishedParsing(Exception):
+	pass
+
+
+@dataclass(frozen=True)
+class PublicationResult:
+	key: str
+	title: str
+	year: int
+	pages: Optional[Tuple[int, int]]
+	authors: List[str] = field(default_factory=list)
+	urls: List[str] = field(default_factory=list)
+
+	@property
+	def cite_key(self) -> str:
+		return CITE_KEY_PREFIX + self.key
+
+	@property
+	def doi(self) -> Optional[str]:
+		for url_str in self.urls:
+			url = urlparse(url_str)
+			if url.hostname is not None and url.hostname.endswith('doi.org'):
+				return url.path[1:]  # Strip leading '/'
+		return None
+
+	@property
+	def is_peer_reviewed(self) -> Optional[bool]:
+		"""
+		Heuristically determine whether a publication is peer reviewed.
+		"""
+
+		# Preprint on arXiv.org
+		if self.key.startswith('journals/corr/abs-'):
+			return False
+
+		# Consider conference proceedings, journal articles, and dissertations
+		# as peer reviewed.
+		if any([
+			self.key.startswith('phd/'),
+			self.key.startswith('conf/'),
+			self.key.startswith('journals/'),
+		]):
+			return True
+
+		return None
+
+	@property
+	def first_page(self) -> Optional[int]:
+		if self.pages is None:
+			return None
+		return self.pages[0]
+
+	@property
+	def last_page(self) -> Optional[int]:
+		if self.pages is None:
+			return None
+		return self.pages[1]
+
+	@classmethod
+	def from_dump(cls, path: Path, keys: Set[str]) -> List['PublicationResult']:
+		parser = xml.sax.make_parser()
+
+		# Enable DTD parsing
+		parser.setFeature(xml.sax.handler.feature_external_ges, True)
+
+		handler = DBLPHandler(keys)
+		parser.setContentHandler(handler)
+		try:
+			parser.parse(path)
+		except FinishedParsing:
+			pass  # Just a workaround to abort SAX parsing if all entries were found
+
+		return handler.publications
+
+	@classmethod
+	def from_api(cls, key: str) -> 'PublicationResult':
+
+		url = f"https://dblp.uni-trier.de/rec/{key}.xml"
+		response = requests.get(url)
+		response.raise_for_status
+
+		parser = xml.sax.make_parser()
+		handler = DBLPHandler({key})
+		parser.setContentHandler(handler)
+		try:
+			parser.parse(io.BytesIO(response.content))
+		except FinishedParsing:
+			pass
+
+		assert 1 == len(handler.publications)
+
+		return handler.publications[0]
+
+	@classmethod
+	def from_search_hit(cls, hit: Dict[str, Any]) -> 'PublicationResult':
+		info = hit['info']
+
+		pages: Optional[Tuple[int, int]] = None
+		if raw_pages := info.get('pages', None):
+			pages = parse_pages(raw_pages)
+
+		# A single author is not a list, d'oh.
+		authors = info.get('authors', dict()).get('author', [])
+		if type(authors) is not list:
+			authors = [authors]
+
+		# TODO Parse URLs ('ee')
+
+		return cls(
+			key=info['key'],
+			title=clean_title(html.unescape(info['title'])),
+			year=int(info['year']),
+			pages=pages,
+			authors=[html.unescape(author['text']) for author in authors],
+		)
+
+	@classmethod
+	def from_search(
+		cls,
+		search_term: str,
+		limit: int = 1000,
+	) -> Tuple[str, List['PublicationResult'], int]:
+		# see https://dblp.uni-trier.de/faq/13501473.html
+		url = 'http://dblp.org/search/publ/api'
+		response = requests.get(
+			url,
+			params={
+				'q': search_term,
+				'f': 0,
+				'h': limit,
+				'c': 0,
+				'format': 'json',
+			},
+		)
+		response.raise_for_status
+		search_result = response.json()['result']
+		hits = search_result['hits']
+		results = [cls.from_search_hit(hit) for hit in hits['hit']]
+
+		total = hits['@total']
+		# TODO re-request if len(results) < hits_total
+
+		return (search_result['query'], results, total)
+
+
+@dataclass
+class DBLPFullHandler(xml.sax.handler.ContentHandler):
+	entries: Dict[str, str] = field(default_factory=dict)
+
+	# ContentHandler
+
+	def startElement(self, name: str, attributes: Attributes):
+		if name in PUBLICATIONS:
+			key = attributes.getValue('key')
+			self.entries[key] = name
+
+
+def get_all_cite_keys(path: Path) -> Set[str]:
+	cache_path = path.with_suffix('.pickle')
+	cache: Dict[str, str] = dict()
+	if cache_path.exists():
+		with cache_path.open('rb') as f:
+			cache = pickle.load(f)
+	else:
+		parser = xml.sax.make_parser()
+
+		# Enable DTD parsing
+		parser.setFeature(xml.sax.handler.feature_external_ges, True)
+
+		handler = DBLPFullHandler()
+		parser.setContentHandler(handler)
+		parser.parse(path)
+
+		cache = handler.entries
+		with cache_path.open('wb') as f:
+			pickle.dump(cache, f)
+
+	return {CITE_KEY_PREFIX + key for key in cache.keys()}
+
+
+@dataclass
+class DBLPHandler(xml.sax.handler.ContentHandler):
+	key_queue: Set[str]
+	tag_stack: List[str] = field(default_factory=list)
+	publications: List[PublicationResult] = field(default_factory=list)
+	key: Optional[str] = None
+	author: Optional[str] = None
+	authors: List[Author] = field(default_factory=list)
+	title: Optional[str] = None
+	year: Optional[int] = None
+	pages: Optional[Tuple[int, int]] = None
+	urls: List[str] = field(default_factory=list)
+
+	@property
+	def current_tag(self) -> str:
+		assert 0 < len(self.tag_stack)
+		return self.tag_stack[-1]
+
+	@property
+	def is_handling_publication(self) -> bool:
+		return self.key is not None
+
+	def startElement(self, name: str, attributes: Attributes):
+		self.tag_stack.append(name)
+
+		if name in PUBLICATIONS:
+			self.startPublication(name, attributes)
+
+		if not self.is_handling_publication:
+			return
+
+		if name == 'author':
+			self.author = ''
+		if name == 'title':
+			self.title = ''
+
+	def endElement(self, name: str):
+		self.tag_stack.pop(-1)
+
+		if self.is_handling_publication and name in PUBLICATIONS:
+			self.endPublication()
+
+		if name == 'author' and self.author is not None:
+			self.authors.append(self.author)
+			self.author = None
+
+	def characters(self, content: Union[bytes, str]):
+		assert isinstance(content, str)  # TODO Handle bytes?
+
+		if not self.is_handling_publication:
+			return
+
+		if 'author' in self.tag_stack:
+			assert self.author is not None
+			self.author += content
+
+		if 'title' in self.tag_stack:
+			assert self.title is not None
+			self.title += content
+
+		if self.current_tag == 'ee':
+			self.urls.append(content)
+
+		if self.current_tag == 'year':
+			assert self.year is None
+			self.year = int(content)
+
+		if self.current_tag == 'pages':
+			assert self.pages is None
+			self.pages = parse_pages(content)
+
+	def startPublication(self, name: str, attributes: Attributes):
+		assert name in PUBLICATIONS
+		assert not self.is_handling_publication
+		assert 'key' in attributes
+		assert self.author is None
+		assert 0 == len(self.authors)
+		assert self.title is None
+		assert self.year is None
+		assert self.pages is None
+		assert 0 == len(self.urls)
+
+		key = attributes.getValue('key')
+		if key not in self.key_queue:
+			return  # This is not the publication you are looking for.
+
+		self.key_queue.remove(key)
+		self.key = key
+
+	def endPublication(self):
+		assert self.is_handling_publication
+		assert self.author is None
+		assert self.key is not None
+		assert self.title is not None
+		assert self.year is not None
+		assert 0 < len(self.authors)
+
+		title = self.title
+		if title.endswith('.'):
+			title = title[:-1]
+
+		publication = PublicationResult(
+			key=self.key,
+			title=title,
+			year=self.year,
+			authors=self.authors,
+			pages=self.pages,
+			urls=self.urls,
+		)
+		self.publications.append(publication)
+
+		self.key = None
+		self.authors = []
+		self.title = None
+		self.year = None
+		self.pages = None
+		self.urls = []
+
+		if 0 == len(self.key_queue):
+			raise FinishedParsing
+
+
+class Command(BaseCommand):
+
+	def log_success(self, msg: str):
+		self.stdout.write(self.style.SUCCESS(msg))
+
+	def log_info(self, msg: str, nl: bool = True):
+		self.stdout.write(self.style.HTTP_INFO(msg), ending='\n' if nl else '')
+		self.stdout.flush()
+
+	# BaseCommand
+
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('--use-api', action='store_true', default=False)
+		parser.add_argument('--search-term', default=None)
+		parser.add_argument('keys', nargs='+')
+
+	@transaction.atomic
+	def handle(self, *args, **options):
+		use_api = options['use_api']
+		source = Source.objects.get(name='DBLP')
+
+		search_term: Optional[SearchTerm] = None
+		if name := options['search_term']:
+			search_term, created = SearchTerm.objects.get_or_create(name=name)
+			if created:
+				self.log_success(f"Created search term: {search_term}")
+
+		cite_keys: Set[str] = set()
+		publications: List[Publication] = []
+		for key in set(options['keys']):
+			try:
+				publication = Publication.objects.get(cite_key=key)
+				publications.append(publication)
+			except Publication.DoesNotExist:
+				if not key.startswith(CITE_KEY_PREFIX):
+					raise CommandError(f"Invalid cite key: {key}")
+				cite_keys.add(strip_cite_key_prefix(key))
+
+		if 0 < len(cite_keys):
+
+			if use_api:
+				self.log_info("Querying DBLP... ", nl=False)
+			else:
+				self.log_info(f"Parsing DBLP dump '{DUMP_PATH}'... ", nl=False)
+			start = datetime.now()
+			if use_api:
+				results: List[PublicationResult] = []
+				for key in cite_keys:
+					result = PublicationResult.from_api(key)
+					results.append(result)
+			else:
+				results = PublicationResult.from_dump(DUMP_PATH, cite_keys)
+			end = datetime.now()
+			duration = end - start
+			self.log_success(f"done ({duration}).")
+
+			for result in results:
+
+				# Add authors to database
+				authors: List[Author] = []
+				for name in result.authors:
+					author, created = Author.objects.get_or_create(name=name)
+					if created:
+						self.log_success(f"Added author: {author}")
+					else:
+						self.log_info(f"Author '{author}' alreay known")
+					authors.append(author)
+
+				# Add publication to database
+				publication, created = Publication.objects.get_or_create(
+					cite_key=result.cite_key,
+					title=result.title,
+					year=result.year,
+					peer_reviewed=result.is_peer_reviewed,
+					first_page=result.first_page,
+					last_page=result.last_page,
+					doi=result.doi,
+				)
+				if created:
+					self.log_success(f"Added publication: {publication}")
+				else:
+					self.log_info(f"Publication '{publication}' already known")
+				publications.append(publication)
+
+				# Assign authors
+				for position, author in enumerate(authors):
+					publication_author, created = PublicationAuthor.objects.get_or_create(
+						author=author,
+						publication=publication,
+						position=position,
+					)
+					if created:
+						self.log_success(f"Assigned author '{author}' to publication '{publication}' at position {position}")
+					else:
+						self.log_info(f"Author '{author}' already assigned to publication '{publication}' at position '{position}'")
+
+		# Assign sources
+		if search_term is not None:
+			for publication in publications:
+				publication_source, created = PublicationSource.objects.get_or_create(
+					source=source,
+					publication=publication,
+					search_term=search_term,
+				)
+				if created:
+					self.log_success(f"Assigned source '{source}' to publication '{publication}' with search term '{search_term}'")
+				else:
+					self.log_info(f"Source '{source}' already assigned to publication '{publication}' with search term '{search_term}'")

sok/management/commands/dblpsearch.py

@@ -0,0 +1,181 @@
+import pickle
+
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+from django.db import transaction
+
+import sok.management.commands.dblpimport as dblp
+
+from sok.management.commands.snowball import semanticscholar
+from sok.models import (
+	Author,
+	Publication,
+	PublicationAuthor,
+	PublicationSource,
+	SearchTerm,
+	SemanticScholar,
+	Source,
+)
+
+
+class Command(BaseCommand):
+
+	def log_success(self, msg: str):
+		self.stdout.write(self.style.SUCCESS(msg))
+
+	def log_info(self, msg: str, nl: bool = True):
+		self.stdout.write(self.style.HTTP_INFO(msg), ending='\n' if nl else '')
+		self.stdout.flush()
+
+	def display_result(self, result: dblp.PublicationResult):
+		self.stdout.write("")
+		self.log_info(result.cite_key)
+		if 0 < len(result.authors):
+			self.stdout.write("  " + ", ".join([name for name in result.authors]))
+		self.log_info("  " + result.title, nl=False)
+		self.stdout.write(f" ({result.year})")
+
+	def add_publication_source(
+		self,
+		publication: Publication,
+		source: Source,
+		search_term: SearchTerm,
+	):
+		publication_source, created = PublicationSource.objects.get_or_create(
+			source=source,
+			publication=publication,
+			search_term=search_term,
+		)
+		if created:
+			self.log_success(f"Assigned source '{source}' to publication '{publication}' with search term '{search_term}'")
+		else:
+			self.log_info(f"Source '{source}' already assigned to publication '{publication}' with search term '{search_term}'")
+
+	@transaction.atomic
+	def store_result(
+		self,
+		result: dblp.PublicationResult,
+		source: Source,
+		search_term: SearchTerm,
+		paper_id: Optional[str],
+	) -> Publication:
+
+		# Store Authors
+		authors: List[Author] = []
+		for name in result.authors:
+			author, created = Author.objects.get_or_create(name=name)
+			if created:
+				self.log_success(f"Added author: {author}")
+			else:
+				self.log_info(f"Author '{author}' alreay known")
+			authors.append(author)
+
+		# Store Publication
+		publication = Publication(
+			cite_key=result.cite_key,
+			title=result.title,
+			year=result.year,
+			peer_reviewed=result.is_peer_reviewed,
+			first_page=result.first_page,
+			last_page=result.last_page,
+		)
+		publication.full_clean()
+		publication.save()
+		self.log_success(f"Added publication: {publication}")
+
+		# Assign authors to publication
+		for position, author in enumerate(authors):
+			publication_author, created = PublicationAuthor.objects.get_or_create(
+				author=author,
+				publication=publication,
+				position=position,
+			)
+			if created:
+				self.log_success(f"Assigned author '{author}' to publication '{publication}' at position {position}")
+			else:
+				self.log_info(f"Author '{author}' already assigned to publication '{publication}' at position '{position}'")
+
+		if paper_id is not None:
+			s, created = SemanticScholar.objects.get_or_create(paper_id=paper_id, publication=publication)
+			if created:
+				self.log_success(f"Added Semantic Scholar '{paper_id}' to publication '{publication}'")
+			else:
+				self.log_info(f"Semantic Scholar '{paper_id}' for publication '{publication}' is already known")
+
+		self.add_publication_source(publication, source, search_term)
+		return publication
+
+	# BaseCommand
+
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('--reset-choices', action='store_true')
+		parser.add_argument('--limit', type=int, default=1000, help="1 – 1000 (default: 1000)")
+		parser.add_argument('term')
+
+	def handle(self, *args, **options):
+		try:
+			limit: int = options['limit']
+			if not (0 < limit <= 1000):
+				raise CommandError(f"Invalid value for 'limit': {limit}; allowed range is 1 – 1000")
+			reset_choices: bool = options['reset_choices']
+			source = Source.objects.get(name='DBLP')
+
+			path = Path('.choices.dblp.pickle')
+			cache: Dict[str, Set[str]] = defaultdict(set)
+			if reset_choices:
+				path.unlink(missing_ok=True)
+			elif path.exists():
+				self.log_info("Loading previous choices (reset with --reset-choices)...", nl=False)
+				with path.open('rb') as f:
+					cache = pickle.load(f)
+				self.log_success("done")
+
+			self.log_info("Querying DBLP... ", nl=False)
+			query, results, total = dblp.PublicationResult.from_search(options['term'], limit)
+			self.log_success(f"done, found {len(results)}/{total} publication(s)")
+
+			# Create search term
+			search_term, created = SearchTerm.objects.get_or_create(name=query)
+			if created:
+				self.log_success(f"Created search term: {search_term}")
+
+			# Add search term to existing entries
+			cite_keys = {result.cite_key for result in results}
+			existing: Set[str] = set()
+			for publication in Publication.objects.filter(cite_key__in=cite_keys):
+				existing.add(publication.cite_key)
+				self.add_publication_source(publication, source, search_term)
+
+			# Promt the user for importing new entries
+			for result in results:
+				# Skip existing entries
+				if result.cite_key in existing.union(cache[query]):
+					continue
+
+				self.display_result(result)
+
+				# TODO Add abstract from semantic scholar
+
+				data: Dict[str, Any] = dict()
+				if doi := result.doi:
+					data = semanticscholar(doi)
+
+				while True:
+					choice = input("Import? [y/N], Show abstract? [a]: ").lower()
+					if choice in {'y', 'yes'}:
+						self.store_result(result, source, search_term, data.get('paperId', None))
+						break
+					elif choice in {'', 'n', 'no'}:
+						# Store choice
+						cache[query].add(result.cite_key)
+						with path.open('wb') as f:
+							pickle.dump(cache, f)
+						break
+					elif choice == 'a':
+						if abstract := data.get('abstract', None):
+							self.stdout.write(abstract)
+		except KeyboardInterrupt:
+			raise CommandError("Aborted.")

sok/management/commands/dblptex.py

@@ -0,0 +1,20 @@
+import requests
+
+from django.core.management.base import BaseCommand, CommandParser
+
+import sok.management.commands.dblpimport as dblp
+
+
+class Command(BaseCommand):
+
+	# BaseCommand
+
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('key')
+
+	def handle(self, *args, **options):
+		key = dblp.strip_cite_key_prefix(options['key'])
+		url = f'https://dblp.uni-trier.de/rec/{key}.bib?param=0'
+		response = requests.get(url)
+		response.raise_for_status
+		self.stdout.write(response.content.decode())

sok/management/commands/repair.py

@@ -0,0 +1,111 @@
+from pprint import pprint
+from time import sleep
+
+from django.db import transaction
+from django.core.exceptions import ValidationError
+from django.core.management.base import BaseCommand, CommandError
+
+import sok.management.commands.dblpimport as dblp
+
+from sok.management.commands.snowball import semanticscholar
+from sok.models import Publication, PublicationReference, SemanticScholar
+
+
+class Command(BaseCommand):
+
+	def log_success(self, msg: str):
+		self.stdout.write(self.style.SUCCESS(msg))
+
+	def log_info(self, msg: str, nl: bool = True):
+		self.stdout.write(self.style.HTTP_INFO(msg), ending='\n' if nl else '')
+		self.stdout.flush()
+
+	@transaction.atomic
+	def fix_references(self) -> None:
+		"""
+		Create relevant references to masters of referenced variants.
+
+		If mulitple variants of a publication exist, only the master variant is
+		considered. However, relevant publications might reference a non-master
+		master-variant, e. g., a preprint.
+
+		This command adds references to the master-variant, even though this
+		reference is not actually present in the publication. The reference
+		identifier is marked with a star, e. g., '[1]*'.
+		"""
+
+		self.log_info("--- Searching for references to variants ---")
+		for publication in Publication.objects.filter(variant_of__isnull=False):
+			variant = publication.variant_of
+			origs = PublicationReference.objects.filter(reference=publication)
+			for orig in origs:
+				if PublicationReference.objects.filter(reference=variant, publication=orig.publication).exists():
+					continue
+				fixed = PublicationReference(
+					reference=variant,
+					publication=orig.publication,
+					identifier=('' if orig.identifier is None else orig.identifier) + "*",
+				)
+				try:
+					fixed.full_clean()
+					fixed.save()
+					self.log_success(f"Added reference: {publication} -- {fixed.identifier} -> {variant}")
+				except ValidationError as e:
+					raise CommandError(f"{publication} -- {fixed.identifier} -> {variant}: {e}")
+
+	def fix_dblp(self):
+		self.log_info("--- Searching for entries not in the default DBLP dump ---")
+		keys_in_db = set(
+			Publication.objects.filter(
+				cite_key__startswith=dblp.CITE_KEY_PREFIX
+			).values_list('cite_key', flat=True).distinct()
+		)
+		keys_in_dump = dblp.get_all_cite_keys(dblp.DUMP_PATH)
+
+		self.stdout.write(f"DB:   {len(keys_in_db):8d}")
+		self.stdout.write(f"DBLP: {len(keys_in_dump):8d}")
+		pprint(keys_in_db - keys_in_dump)
+
+	def find_missing_dois(self):
+		self.log_info("--- Searching for missing DOIs ---")
+		publications = Publication.objects.filter(doi__isnull=True)
+		keys = {
+			dblp.strip_cite_key_prefix(cite_key)
+			for cite_key in publications.values_list('cite_key', flat=True)
+		}
+		self.log_info("Parsing DBLP dump...")
+		results = dblp.PublicationResult.from_dump(dblp.DUMP_PATH, keys)
+		self.log_info("done")
+
+		for result in results:
+			if doi := result.doi:
+				publication = publications.get(cite_key=result.cite_key)
+				publication.doi = doi
+				publication.full_clean()
+				publication.save()
+				self.log_success(f"Added DOI '{doi}' to publication: {publication}")
+
+	def find_semanticscholar_ids(self):
+		self.log_info("--- Searching for paper IDs on Semantic Scholar ---")
+		publications = Publication.objects.filter(
+			doi__isnull=False,
+			semanticscholar__isnull=True,
+		)
+		for publication in publications:
+			data = semanticscholar(publication.doi)
+
+			paper_id = data['paperId']
+			obj = SemanticScholar(paper_id=paper_id, publication=publication)
+			obj.full_clean()
+			obj.save()
+			self.log_success(f"Set semanticscholar ID for publication '{publication}': {paper_id}")
+
+			sleep(2)  # Throttle to avoid rate-limiting
+
+	# BaseCommand
+
+	def handle(self, *args, **options):
+		self.fix_references()
+		self.fix_dblp()
+		self.find_missing_dois()
+		self.find_semanticscholar_ids()

sok/management/commands/snowball.py

@@ -0,0 +1,202 @@
+import hashlib
+import json
+import pickle
+
+from pathlib import Path
+from time import sleep
+from typing import Any, Dict, List, Set
+
+import requests
+
+from django.core.management.base import BaseCommand, CommandParser, CommandError
+from tqdm import tqdm
+
+from sok.models import Publication, PublicationReference, SemanticScholar
+
+
+def semanticscholar(identifier: str, include_unknown_references: bool = False) -> Dict[str, Any]:
+	"""
+	Retrieve information from the Semantic Scholar API.
+
+	The identifier can be a DOI or the Semantic Scholar paper ID.
+
+	See: https://api.semanticscholar.org
+	"""
+
+	url = f'https://api.semanticscholar.org/v1/paper/{identifier}'
+	params: Dict[str, Any] = dict()
+	if include_unknown_references:
+		params['include_unknown_references'] = 'true'
+	response = requests.get(url, params=params)
+	response.raise_for_status
+	return response.json()
+
+
+class Command(BaseCommand):
+
+	def echo(self, msg: str, bold: bool = False, nl: bool = True):
+		if bold:
+			msg = self.style.HTTP_INFO(msg)
+		tqdm.write(msg, end='\n' if nl else '')
+		#self.stdout.write(msg, ending='\n' if nl else '')
+
+	def warn(self, msg: str):
+		self.echo(self.style.WARNING(msg))
+
+	def add_reference(
+		self,
+		publication: Publication,
+		reference: Publication,
+		is_reference: bool = True,
+	):
+		try:
+			rel = PublicationReference.objects.get(
+				publication=publication,
+				reference=reference,
+			)
+			if is_reference:
+				self.echo(f"Reference already known: {rel.identifier} {reference}")
+			else:
+				self.echo(f"Citation already known: {rel.identifier} {publication}")
+		except PublicationReference.DoesNotExist:
+			rel = PublicationReference(
+				publication=publication,
+				reference=reference,
+			)
+			rel.full_clean()
+			rel.save()
+			if is_reference:
+				self.echo(f"Added reference: {reference}")
+			else:
+				self.echo(f"Added citation: {publication}")
+
+	def display(self, obj: Dict[str, Any]):
+		self.echo("")
+		authors = [author['name'] for author in obj['authors']]
+		title = obj['title']
+		self.echo("  " + ", ".join(authors))
+		self.echo(f"  {title}", bold=True, nl=False)
+		if year := obj.get('year', None):
+			self.echo(f" ({year})")
+		else:
+			self.echo("")
+		if venue := obj.get('venue', None):
+			self.echo(f"  {venue}")
+		if doi := obj.get('doi', None):
+			self.echo(f"  {doi}")
+		if paper_id := obj.get('paperId', None):
+			self.echo(f"  {paper_id}")
+
+	def get_identifier(self, obj: Dict[str, Any]) -> str:
+		if paper_id := obj.get('paperId', None):
+			return paper_id
+		raw = json.dumps(obj, sort_keys=True)
+		hasher = hashlib.blake2b()
+		hasher.update(raw.encode())
+		return hasher.hexdigest()
+
+	def handle_objs(
+		self,
+		base: Publication,
+		objs: List[Dict[str, Any]],
+		is_reference: bool,
+	):
+		title = "Reference" if is_reference else "Citation"
+		if 0 < len(objs):
+			self.echo(f"--- {title}s ---")
+		for obj in tqdm(objs, unit=title.lower()):
+			if paper_id := obj.get('paperId', None):
+				try:
+					existing = SemanticScholar.objects.get(paper_id=paper_id)
+					if is_reference:
+						self.add_reference(base, existing.publication)
+					else:
+						self.add_reference(existing.publication, base, is_reference)
+					continue
+				except SemanticScholar.DoesNotExist:
+					if doi := obj.get('doi', None):
+						try:
+							publication = Publication.objects.get(doi=doi)
+							new = SemanticScholar(paper_id=paper_id, publication=publication)
+							new.full_clean()
+							new.save()
+							self.echo(f"New Semantic Scholar entry: {paper_id}")
+							if is_reference:
+								self.add_reference(base, new.publication)
+							else:
+								self.add_reference(new.publication, base, is_reference)
+							continue
+						except Publication.DoesNotExist:
+							pass
+
+			identifier = self.get_identifier(obj)
+			if identifier in self.cache:
+				continue
+
+			self.display(obj)
+
+			paper_id = obj.get('paperId', None)
+			while True:
+				self.echo("Ignore? [Y/n]", nl=False)
+				if paper_id is not None:
+					self.echo(", Show abstract [a]", nl=False)
+				self.echo(": ")
+				choice = input().lower()
+				if choice in {'', 'y', 'yes'}:
+					# Store choice
+					self.cache.add(identifier)
+					with self.cache_path.open('wb') as f:
+						pickle.dump(self.cache, f)
+					break
+				elif choice in {'a'}:
+					assert paper_id is not None
+					data = semanticscholar(paper_id)
+					if abstract := data.get('abstract', None):
+						self.echo(abstract)
+				elif choice in {'', 'n', 'no'}:
+					# TODO Import?
+					break
+
+	# BaseCommand
+
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('--reset-choices', action='store_true')
+		parser.add_argument('--no-references', action='store_true')
+		parser.add_argument('--no-citations', action='store_true')
+
+	def handle(self, *args, **options):
+		reset_choices: bool = options['reset_choices']
+		no_citations: bool = options['no_citations']
+		no_references: bool = options['no_references']
+
+		self.cache_path = Path('.choices.semanticscholar.pickle')
+		self.cache: Set[str] = set()
+		if reset_choices:
+			self.cache_path.unlink(missing_ok=True)
+		elif self.cache_path.exists():
+			self.echo("Loading previous choices (reset with --reset-choices)...", nl=False)
+			with self.cache_path.open('rb') as f:
+				self.cache = pickle.load(f)
+			self.echo("done", bold=True)
+
+		publications = Publication.objects.filter(
+			semanticscholar__isnull=False,
+			exclusion_criteria__isnull=True,
+		)
+		try:
+			for publication in tqdm(publications, unit="publication"):
+				self.echo(f"=== Publication {publication} ===")
+				for semantic in publication.semanticscholar_set.all():
+					data = semanticscholar(semantic.paper_id)
+
+					if not no_references:
+						references: List[Dict[str, Any]] = data['references']
+						self.handle_objs(publication, references, is_reference=True)
+
+					if not no_citations:
+						citations: List[Dict[str, Any]] = data['citations']
+						self.handle_objs(publication, citations, is_reference=False)
+
+					sleep(2)  # Throttle
+		except KeyboardInterrupt:
+			raise CommandError("Aborted.")

sok/management/commands/stats.py

@@ -0,0 +1,49 @@
+from typing import Set
+
+from django.core.management.base import BaseCommand
+from django.db.models import Count, Q
+
+import sok.management.commands.dblpimport as dblp
+
+from sok.models import Publication, SearchTerm
+
+
+class Command(BaseCommand):
+
+	def echo(self, msg: str, bold=True):
+		if bold:
+			msg = self.style.HTTP_INFO(msg)
+		self.stdout.write(msg)
+
+	# BaseCommand
+
+	def handle(self, *args, **options):
+		publications_found: Set[str] = set()
+		publications_peer_reviewed: Set[str] = set()
+		publications_relevant: Set[str] = set()
+
+		self.echo("Loading DBLP dump...")
+		all_cite_keys = dblp.get_all_cite_keys(dblp.DUMP_PATH)
+
+		for search_term in SearchTerm.objects.all():
+			# DBLP search result
+			self.echo(f"Searching DBLP for '{search_term}'")
+			query, results, total = dblp.PublicationResult.from_search(search_term.name, 1000)
+			for result in results:
+				if result.cite_key not in all_cite_keys:
+					continue
+				publications_found.add(result.cite_key)
+				if result.is_peer_reviewed:
+					publications_peer_reviewed.add(result.cite_key)
+
+			# Relevant publications
+			for publication in Publication.objects.filter(
+				publicationsource__search_term=search_term,
+				exclusion_criteria__isnull=True,
+			).distinct():
+				publications_relevant.add(publication.cite_key)
+
+		# Output
+		self.echo(f"Total publications: {len(publications_found):4d}", bold=True)
+		self.echo(f"- peer reviewed:    {len(publications_peer_reviewed):4d}", bold=True)
+		self.echo(f"- relevant:         {len(publications_relevant):4d}", bold=True)

sok/management/commands/tagdag.py

@@ -0,0 +1,38 @@
+from typing import Set, Tuple
+
+from django.core.management.base import BaseCommand, CommandParser
+
+from sok.models import Tag
+
+
+class Command(BaseCommand):
+
+	def echo(self, msg: str):
+		self.stdout.write(msg)
+
+	# BaseCommand
+
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('--root', default='CAPI Misuse')
+
+	def _graphviz(self, root: Tag) -> None:
+		for tag in root.implied_by.all():
+			edge = (tag.pk, root.pk)
+			if edge in self.graph:
+				continue
+			if edge[::-1] in self.graph:
+				self.stderr.write(self.style.ERROR(f"CYCLE: '{root}' <-> '{tag}'"))
+			self.graph.add(edge)
+			self._graphviz(tag)
+			self.echo(f'\t"{tag}" -> "{root}";')
+
+	def graphviz(self, root: Tag) -> None:
+		self.echo("digraph G {")
+		self.echo("\trankdir = RL;")
+		self._graphviz(root)
+		self.echo("}")
+
+	def handle(self, *args, **options) -> None:
+		self.graph: Set[Tuple[int, int]] = set()
+		root = Tag.objects.get(name=options['root'])
+		self.graphviz(root)