Fix snowballing functionality

sok/admin.py

@@ -64,6 +64,7 @@ class PublicationStageFilter(admin.SimpleListFilter):
 		return (
 			('primary', _("primary")),
 			('secondary', _("secondary")),
+			('2-secondary', _("2-secondary")),
 			('tertiary', _("tertiary")),
 			('excluded', _("excluded")),
 			('-', _("-")),
@@ -95,6 +96,14 @@ class PublicationStageFilter(admin.SimpleListFilter):
 				referenced_by__sources__isnull=False,
 			)
 
+		if self.value() == '2-secondary':
+			ids: Set[int] = {
+				publication.id
+				for publication in queryset
+				if publication.stage == '2-secondary'
+			}
+			return queryset.filter(id__in=ids)
+
 		if self.value() == '-':
 			ids: Set[int] = {
 				publication.id

sok/management/commands/dblpimport.py

@@ -96,7 +96,8 @@ class PublicationResult:
 	year: int
 	pages: Optional[Tuple[int, int]]
 	dblp_doi: Optional[str] = None
-	authors: List[str] = field(default_factory=list)
+	access: Optional[str] = None
+	authors: List[Tuple[str,str]] = field(default_factory=list)
 	urls: List[str] = field(default_factory=list)
 
 	@property
@@ -166,7 +167,7 @@ class PublicationResult:
 
 		url = f"https://dblp.uni-trier.de/rec/{key}.xml"
 		response = requests.get(url)
-		response.raise_for_status
+		response.raise_for_status()
 
 		parser = xml.sax.make_parser()
 		handler = DBLPHandler({key})
@@ -195,6 +196,7 @@ class PublicationResult:
 
 		# TODO Parse URLs ('ee')
 		doi = None if not 'doi' in info else info['doi']
+		access = None if not 'access' in info else info['access']
 
 		return cls(
 			key=info['key'],
@@ -202,7 +204,8 @@ class PublicationResult:
 			year=int(info['year']),
 			pages=pages,
 			dblp_doi=doi,
-			authors=[html.unescape(author['text']) for author in authors],
+			access = access,
+			authors=[(author['@pid'], html.unescape(author['text'])) for author in authors],
 		)
 
 	@classmethod
@@ -223,10 +226,10 @@ class PublicationResult:
 				'format': 'json',
 			},
 		)
-		response.raise_for_status
+		response.raise_for_status()
 		search_result = response.json()['result']
 		hits = search_result['hits']
-		if hits['@total'] == '0': return (None, None, 0)
+		if hits['@total'] == '0' or hits['@sent'] == '0': return (None, None, 0)
 		results = [cls.from_search_hit(hit) for hit in hits['hit']]
 
 		total = hits['@total']
@@ -453,6 +456,7 @@ class Command(BaseCommand):
 				# Add authors to database
 				authors: List[Author] = []
 				for name in result.authors:
+					# TODO? find author id's -> not in xml!
 					author, created = Author.objects.get_or_create(name=name)
 					if created:
 						self.log_success(f"Added author: {author}")

sok/management/commands/dblpsearch.py

@@ -34,7 +34,7 @@ class Command(BaseCommand):
 		self.stdout.write("")
 		self.log_info(result.cite_key)
 		if 0 < len(result.authors):
-			self.stdout.write("  " + ", ".join([name for name in result.authors]))
+			self.stdout.write("  " + ", ".join([author.name for author in result.authors]))
 		self.log_info("  " + result.title, nl=False)
 		self.stdout.write(f" ({result.year})")
 
@@ -66,8 +66,16 @@ class Command(BaseCommand):
 
 		# Store Authors
 		authors: List[Author] = []
-		for name in result.authors:
+		for (pid, name) in result.authors:
 			author, created = Author.objects.get_or_create(name=name)
+			if not author.dblp_id and pid != None:
+				try:
+					author.dblp_id = pid
+					author.full_clean()
+					author.save()
+				except ValidationError as e:
+					raise CommandError(f"{author}: {e}")
+
 			if created:
 				self.log_success(f"Added author: {author}")
 			else:

sok/management/commands/repair.py

@@ -2,14 +2,19 @@ from pprint import pprint
 from time import sleep
 from tqdm import tqdm
 
+import re, requests, html
+
 from django.db import transaction
 from django.core.exceptions import ValidationError
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+from django.db.models import Count
+
+from typing import List, Optional, Tuple
 
 import sok.management.commands.dblpimport as dblp
 
 from sok.management.commands.snowball import semanticscholar
-from sok.models import Publication, PublicationReference, SemanticScholar
+from sok.models import Publication, PublicationReference, SemanticScholar, Author, PublicationAuthor
 
 
 class Command(BaseCommand):
@@ -17,6 +22,9 @@ class Command(BaseCommand):
 	def log_success(self, msg: str):
 		self.stdout.write(self.style.SUCCESS(msg))
 
+	def log_warn(self, msg: str):
+		self.stdout.write(self.style.WARNING(msg))
+
 	def log_info(self, msg: str, nl: bool = True):
 		self.stdout.write(self.style.HTTP_INFO(msg), ending='\n' if nl else '')
 		self.stdout.flush()
@@ -71,6 +79,7 @@ class Command(BaseCommand):
 				data = semanticscholar(semantic.paper_id)
 				if abstract := data['abstract']:
 					publication.abstract = abstract
+					if publication.peer_reviewed == None: publication.peer_reviewed = False
 					try:
 						publication.full_clean()
 						publication.save()
@@ -83,30 +92,79 @@ class Command(BaseCommand):
 	def search_on_dblp(self, publication: Publication):
 		query, results, total = dblp.PublicationResult.from_search(publication.title, 100)
 		if total == 0: return
+		plausible = []
 		for result in results:
 			if publication.doi and result.doi:
 				if publication.doi.lower() == result.doi.lower():
-					publication.cite_key = result.cite_key
-					publication.year = result.year
-					if not result.is_peer_reviewed == None:
-						publication.peer_reviewed = result.is_peer_reviewed
-					publication.first_page = result.first_page
-					publication.last_page = result.last_page
-					print(publication.peer_reviewed, result)
-					# keep authors from semantic scholar for now, even though they might be a little broken?
-					try:
-						publication.full_clean()
-						publication.save()
-						self.log_success(f"Added DBLP info for: {publication}")
-					except ValidationError as e:
-						raise CommandError(f"{publication}: {e}")
+					plausible = [result]
+					break
 				else: continue # quite definitely not the same publication
 			elif publication.title == result.title and publication.year == result.year:
-				print(f"Not quite certain about {result.cite_key} for publication {publication.title}")
+				plausible.append(result)
+				self.log_warn(f"Not quite certain about {result.cite_key} for publication {publication.title}")
 			else: continue # I'd rather look at that in detail for now
+			if len(plausible) == 1:
+				result = plausible[0]
+				if result.access.lower() == 'withdrawn':
+					self.log_warn(f"Publication {publication.title} was WITHDRAWN!")
+					return
+				publication.cite_key = result.cite_key
+				publication.year = result.year
+				if not result.is_peer_reviewed == None:
+					publication.peer_reviewed = result.is_peer_reviewed
+				publication.first_page = result.first_page
+				publication.last_page = result.last_page
+				print(publication.peer_reviewed, result)
+				try:
+					publication.full_clean()
+					publication.save()
+					self.log_success(f"Added DBLP info for: {publication}")
+				except ValidationError as e:
+					raise CommandError(f"{publication}: {e}")
+
+				# Store Authors
+				authors: List[Author] = []
+				for (pid, name) in result.authors:
+					author, created = Author.objects.get_or_create(name=name)
+
+					if created:
+						self.log_success(f"Added author: {author}")
+					else:
+						self.log_info(f"Author '{author}' already known")
+					if not author.dblp_id and pid != None:
+						try:
+							author.dblp_id = pid
+							author.full_clean()
+							author.save()
+							self.log_success(f"Added pid to author: {author}")
+						except ValidationError as e:
+							raise CommandError(f"{author}: {e}")
+
+					authors.append(author)
+
+				# Replace authors for publication
+				if len(authors) != len(PublicationAuthor.objects.filter(publication=publication)): print(authors, [pa.author for pa in PublicationAuthor.objects.filter(publication=publication)])
+				assert len(authors) >= len(PublicationAuthor.objects.filter(publication=publication))
+				for publication_author in PublicationAuthor.objects.filter(publication=publication):
+					self.log_warn(f"Will now try to delete {publication_author.author} from {publication}")
+					try:
+						publication_author.delete()
+					except e:
+						raise CommandError(f"{publication} - {author}: {e}")
+
+				for position, author in enumerate(authors):
+					publication_author, created = PublicationAuthor.objects.get_or_create(
+						author=author,
+						publication=publication,
+						position=position,
+					)
+					if created:
+						self.log_success(f"Assigned author '{author}' to publication '{publication}' at position {position}")
+					else:
+						self.log_info(f"Author '{author}' already assigned to publication '{publication}' at position '{position}'")
 
 	def find_secondary_on_dblp(self):
-		self.log_info("--- Searching for secondary and tertiary sources in the default DBLP dump ---")
+		self.log_info("--- Searching for snowballed sources on DBLP ---")
 		for publication in tqdm(Publication.objects.exclude(cite_key__startswith=dblp.CITE_KEY_PREFIX), unit="publication"):
 			if publication.stage == 'excluded': continue
 			self.search_on_dblp(publication)
@@ -117,7 +175,7 @@ class Command(BaseCommand):
 			).values_list('cite_key', flat=True).distinct()
 		)
 		n = len(keys)
-		self.log_info(f"Found {n} publications that still need to be checked")
+		self.log_info(f"Found {n} publications that still need to be verified")
 
 	def fix_dblp(self):
 		self.log_info("--- Searching for entries not in the default DBLP dump ---")
@@ -165,16 +223,156 @@ class Command(BaseCommand):
 				obj.full_clean()
 				obj.save()
 				self.log_success(f"Set semanticscholar ID for publication '{publication}': {paper_id}")
-			else: print(publication)
+			else: 
+				self.log_warn(f"Could not find semanticscholar ID for publication '{publication.title}' with DOI '{publication.doi}'")
 
-			sleep(2)  # Throttle to avoid rate-limiting
+			sleep(1)  # Throttle to avoid rate-limiting (1/s with API Key)
+
+	def find_author_on_dblp(
+		self,
+		name: str,
+		limit: int = 100,
+		) -> List[Tuple[str, str]]:
+		url = 'http://dblp.org/search/author/api'
+		response = requests.get(
+			url,
+			params={
+				'q': name,
+				'f': 0,
+				'h': limit,
+				'c': 0,
+				'format': 'json',
+			},
+		)
+		response.raise_for_status()
+		search_result = response.json()['result']
+		hits = search_result['hits']
+		if hits['@total'] == '0': return None
+		results = [hit['info'] for hit in hits['hit']]
+
+		return results
+
+	def verify_author(self, pid, keys):
+		url = f"https://dblp.org/pid/{pid}.xml"
+		response = requests.get(url)
+		response.raise_for_status()
+		pubs = re.findall('key="([^"]*)"', str(response.content))
+		sleep(1)  # Throttle to avoid rate-limiting
+		return keys.intersection(pubs) == keys
+
+	def find_pid_for_authors(self):
+		self.log_info("--- Adding ID's to authors ---")
+		authors = Author.objects.filter(dblp_id__exact="")
+		for author in tqdm(authors, unit='author'):
+			pid = ""
+			keys = set({
+				dblp.strip_cite_key_prefix(cite_key)
+				for cite_key in PublicationAuthor.objects.filter(
+					author__exact=author,
+					publication__cite_key__startswith=dblp.CITE_KEY_PREFIX
+				).values_list('publication__cite_key', flat=True).distinct()}
+			)
+			if len(keys) == 0: continue
+			options = self.find_author_on_dblp(author.name)
+			if not options:
+				self.log_warn(f"Could not find ID for author {author}")
+			elif len(options) == 1:
+				pid = re.match('https://dblp.org/pid/(.*)', options[0]['url']).groups()[0]
+				if self.verify_author(pid, keys):
+					self.log_success(f"Found {pid} for author {author}")
+					author.dblp_id = pid
+					author.full_clean()
+					author.save()
+					self.log_success(f"Added pid for author {author}")
+			else:
+				for opt in options:
+					if not 'url' in opt: continue
+					pid = re.match('https://dblp.org/pid/(.*)', opt['url']).groups()[0]
+					if self.verify_author(pid, keys):
+						self.log_success(f"Found {pid} for author {author}")
+						author.dblp_id = pid
+						author.full_clean()
+						author.save()
+						self.log_success(f"Added pid for author {author}")
+						break
+
+	@transaction.atomic
+	def merge(self, author, variant):
+		for pub_author in PublicationAuthor.objects.filter(author=variant):
+			publication = pub_author.publication
+			if PublicationAuthor.objects.filter(author=author, publication=publication).exists():
+				self.log_warn(f"{author} is already author of {publication}")
+				break
+			fixed = PublicationAuthor(
+				author=author,
+				publication=publication,
+				position=pub_author.position
+			)
+			try:
+				pub_author.delete()
+				fixed.full_clean()
+				fixed.save()
+				self.log_success(f"Changed author: {publication}: {variant} -> {fixed.author}")
+			except ValidationError as e:
+				raise CommandError(f"{publication}: {variant} -> {fixed.author}: {e}")
+		assert not PublicationAuthor.objects.filter(author=variant).exists()
+		variant.delete()
+
+
+
+	def merge_authors(self) -> None:
+		self.log_info("--- Searching for potential duplicate authors and merging their publications ---")
+
+		dup_ids = Author.objects.exclude(dblp_id__exact="").values_list('dblp_id').annotate(count=Count('id')).filter(count__gt=1)
+
+		try:
+			for (pid, counter) in tqdm(dup_ids, unit="duplicate"):
+
+				response = requests.get(f"https://dblp.org/pid/{pid}.xml")
+				response.raise_for_status()
+				dblp_person = re.search('dblpperson name="([^"]*)"', response.content.decode('ascii')).groups()
+				assert len(dblp_person) == 1
+				dblp_name = html.unescape(dblp_person[0])
+
+				variants = Author.objects.filter(dblp_id__exact=pid).exclude(name__exact=dblp_name)
+				# for v in variants:
+				# 	print(v.name, v.dblp_id)
+				# break
+				orig = Author.objects.filter(dblp_id__exact=pid, name__exact=dblp_name)
+				print(variants, dblp_name, orig)
+				assert len(orig) == 1
+
+				self.log_info(f"Suggestion to merge {', '.join([v.name for v in variants])} into {orig[0]}")
+				
+				while True:
+					self.log_warn("Merge? [Y/n]")
+					choice = input().lower()
+					if choice in {'', 'y', 'yes'}:
+						for variant in variants: self.merge(orig[0], variant)
+						break
+					elif choice in {'', 'n', 'no'}:
+						break
+		except KeyboardInterrupt:
+			raise CommandError("Aborted.")
 
 	# BaseCommand
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('-b', '--abstract', action='store_true')
+		parser.add_argument('-d', '--dblp', action='store_true')
+		parser.add_argument('-a', '--authors', action='store_true')
+		parser.add_argument('-s', '--secondary', action='store_true')
+		parser.add_argument('-i', '--scholarid', action='store_true')
 
 	def handle(self, *args, **options):
 		self.fix_references()
-		self.fix_abstracts()
-		self.find_secondary_on_dblp()
-		self.fix_dblp()
-		self.find_missing_dois()
-		self.find_semanticscholar_ids()
+		if options['secondary']: self.find_secondary_on_dblp()
+		if options['dblp']:
+			self.fix_dblp()
+			self.find_missing_dois()
+		if options['scholarid'] or options['dblp']:
+			self.find_semanticscholar_ids()
+		if options['authors']:
+			self.find_pid_for_authors()
+			self.merge_authors()
+		if options['abstract']: self.fix_abstracts()
+

sok/management/commands/snowball.py

@@ -6,6 +6,8 @@ from pathlib import Path
 from time import sleep
 from typing import Any, Dict, List, Set
 
+from django.conf import settings
+
 import requests
 
 from django.core.management.base import BaseCommand, CommandParser, CommandError
@@ -24,7 +26,7 @@ from sok.models import (
 )
 
 
-def semanticscholar(identifier: str, include_unknown_references: bool = False) -> Dict[str, Any]:
+def semanticscholar(identifier: str, fields: str = None, type: str = None, offset: int = 0, include_unknown_references: bool = False) -> Dict[str, Any]:
 	"""
 	Retrieve information from the Semantic Scholar API.
 
@@ -33,11 +35,24 @@ def semanticscholar(identifier: str, include_unknown_references: bool = False) -
 	See: https://api.semanticscholar.org
 	"""
 
-	url = f'https://api.semanticscholar.org/v1/paper/{identifier}'
+	url = f'https://api.semanticscholar.org/graph/v1/paper/{identifier}'
 	params: Dict[str, Any] = dict()
+
+	if type in ['citations', 'references']:
+		url += ('/' + type)
+		params['limit'] = '200'
+		params['offset'] = str(offset)
+
 	if include_unknown_references:
 		params['include_unknown_references'] = 'true'
-	response = requests.get(url, params=params)
+	if fields:
+		params['fields'] = fields
+	
+	headers = {
+			'x-api-key': settings.SCHOLAR_API_KEY
+			}
+
+	response = requests.get(url, params=params, headers=headers)
 	response.raise_for_status
 	return response.json()
 
@@ -48,7 +63,6 @@ class Command(BaseCommand):
 		if bold:
 			msg = self.style.HTTP_INFO(msg)
 		tqdm.write(msg, end='\n' if nl else '')
-		#self.stdout.write(msg, ending='\n' if nl else '')
 
 	def warn(self, msg: str):
 		self.echo(self.style.WARNING(msg))
@@ -113,8 +127,9 @@ class Command(BaseCommand):
 		base: Publication,
 		is_reference: bool,
 	) -> Publication:
-			data = semanticscholar(paper_id)
+			data = semanticscholar(paper_id, fields='title,authors,year,abstract,externalIds')
 			# Add authors to database
+			## TODO: semantic scholar author ids nutzen?
 			authors: List[Author] = []
 			first = True
 			cite_key = ''
@@ -124,12 +139,12 @@ class Command(BaseCommand):
 					if created:
 						self.echo(f"Added author: {author}")
 					else:
-						self.echo(f"Author '{author}' alreay known")
+						self.echo(f"Author '{author}' already known")
 					authors.append(author)
-				cite_key = ''
-				if authors:
-					cite_key = authors[0].name.split(' ')[-1]
-				cite_key += str(data.get('year'))
+			cite_key = ''
+			if authors:
+				cite_key = authors[0].name.split(' ')[-1]
+			cite_key += str(data.get('year'))
 
 			title = data.get('title', '')
 			cite_key += title.split(' ')[0]
@@ -145,22 +160,22 @@ class Command(BaseCommand):
 				publication = None
 
 			# Add publication to database
-			doi = data.get('doi', None)
+			doi = data.get('externalIds', None).get('DOI', None)
 			if not publication:
-				self.echo(f"Will create now with cite key {cite_key}")
 				publication = Publication.objects.create(
 					cite_key=cite_key,
 					title=title,
 					year=data.get('year', 0),
 					peer_reviewed=None,
 					doi=doi,
+					abstract=data.get('abstract', None),
 				)
 				self.echo(f"Added publication: {publication}")
 			else:
 				self.echo(f"Publication '{publication}' already known")
 
 			# Assign authors
-			for position, author in enumerate(authors):
+			for position, author in enumerate(list(set(authors))):
 				publication_author, created = PublicationAuthor.objects.get_or_create(
 					author=author,
 					publication=publication,
@@ -192,9 +207,11 @@ class Command(BaseCommand):
 		title = "Reference" if is_reference else "Citation"
 		if 0 < len(objs):
 			self.echo(f"--- {title}s ---")
-		for obj in tqdm(objs, unit=title.lower()):
+		progress_iterator = tqdm(objs, unit=title.lower(), position=2, leave=False, desc=title + 's')
+		for obj in progress_iterator:
 			if paper_id := obj.get('paperId', None):
 				try:
+					# This publication already exists and has a semantic scholar entry
 					existing = SemanticScholar.objects.get(paper_id=paper_id)
 					if is_reference:
 						self.add_reference(base, existing.publication)
@@ -202,6 +219,7 @@ class Command(BaseCommand):
 						self.add_reference(existing.publication, base, is_reference)
 					continue
 				except SemanticScholar.DoesNotExist:
+					# This publication already exists but does not have a semantic scholar entry
 					if doi := obj.get('doi', None):
 						try:
 							publication = Publication.objects.get(doi=doi)
@@ -215,6 +233,7 @@ class Command(BaseCommand):
 								self.add_reference(new.publication, base, is_reference)
 							continue
 						except Publication.DoesNotExist:
+						# This publication does not exist so we need to create it
 							pass
 
 			identifier = self.get_identifier(obj)
@@ -225,11 +244,12 @@ class Command(BaseCommand):
 
 			paper_id = obj.get('paperId', None)
 			while True:
-				self.echo("Ignore? [Y/n]", nl=True)
-				if paper_id is not None:
+				self.echo("Ignore? [Y/n]", nl=False)
+				if paper_id:
 					self.echo(", Show abstract [a]", nl=False)
 				self.echo(": ")
 				choice = input().lower()
+
 				if choice in {'', 'y', 'yes'}:
 					# Store choice
 					self.cache.add(identifier)
@@ -238,15 +258,16 @@ class Command(BaseCommand):
 					break
 				elif choice in {'a'}:
 					assert paper_id is not None
-					data = semanticscholar(paper_id)
-					if abstract := data.get('abstract', None):
+					if abstract := obj.get('abstract', None):
 						self.echo(abstract)
+					else:
+						self.echo('Sorry, there is no abstract for this publication on Semantic Scholar')
 				elif choice in {'', 'n', 'no'}:
-					# TODO Import? copied and adapted from PR
+					# DONE Import? copied and adapted from PR
 					if paper_id is not None:
 						self.add_publ(paper_id, base, is_reference)
 					else:
-						self.echo("Could not add this paper, please do it manually!")
+						self.warn("Could not add this paper, please do it manually!")
 					break
 
 	# BaseCommand
@@ -255,11 +276,13 @@ class Command(BaseCommand):
 		parser.add_argument('--reset-choices', action='store_true')
 		parser.add_argument('--no-references', action='store_true')
 		parser.add_argument('--no-citations', action='store_true')
+		parser.add_argument('-s', '--stage', type=int, default=10000)
 
 	def handle(self, *args, **options):
 		reset_choices: bool = options['reset_choices']
 		no_citations: bool = options['no_citations']
 		no_references: bool = options['no_references']
+		stage: int = options['stage']
 
 		self.cache_path = Path('.choices.semanticscholar.pickle')
 		self.cache: Set[str] = set()
@@ -275,20 +298,44 @@ class Command(BaseCommand):
 			semanticscholar__isnull=False,
 			exclusion_criteria__isnull=True,
 		)
+
+		if stage < 10000:
+			publications = [p for p in publications if p.stage_added() == stage]
+			self.echo(f"==== {len(publications)} publications from stage {stage} will be shown ====")
 		try:
-			for publication in tqdm(publications, unit="publication"):
-				self.echo(f"=== Publication {publication} ===")
+			progress_iterator = tqdm(publications, unit="publication", position=1, desc='Publications')
+			for publication in progress_iterator:
+				self.echo(f"=== Publication {publication}: {publication.title} ===")
 				for semantic in publication.semanticscholar_set.all():
-					data = semanticscholar(semantic.paper_id)
 
 					if not no_references:
-						references: List[Dict[str, Any]] = data['references']
-						self.handle_objs(publication, references, is_reference=True)
+						offset = 0
+						while True:
+							data = semanticscholar(semantic.paper_id, type='references', fields='title,abstract', offset=offset)
+							if not data.get('data', None):
+								self.echo(self.style.WARNING("API did not return any references, verify manually!"))
+								break
+							references: List[Dict[str, Any]] = [d['citedPaper'] for d in data['data']]
+							self.handle_objs(publication, references, is_reference=True)
+							# Handle limitation if there are more than 200 paper references
+							if 'next' in data: offset = data['next']
+							else: break
 
 					if not no_citations:
-						citations: List[Dict[str, Any]] = data['citations']
-						self.handle_objs(publication, citations, is_reference=False)
+						offset = 0
+						while True:
+							data = semanticscholar(semantic.paper_id, type='citations', fields='title,abstract', offset=offset)
+							if not data.get('data', None):
+								self.echo(self.style.WARNING("API did not return any citations, verify manually!"))
+								break
+							citations: List[Dict[str, Any]] = [d['citingPaper'] for d in data['data']]
+							self.handle_objs(publication, citations, is_reference=False)
+							# Handle limitation if there are more than 200 paper references
+							if 'next' in data: offset = data['next']
+							else: break
+
+					sleep(1)  # Throttle
 
-					sleep(2)  # Throttle
 		except KeyboardInterrupt:
 			raise CommandError("Aborted.")
+

sok/management/commands/stats.py

@@ -1,4 +1,8 @@
 from typing import Set
+import pickle
+from django.db.models import Count
+
+from pathlib import Path
 
 from django.core.management.base import BaseCommand
 from django.db.models import Count, Q
@@ -43,7 +47,22 @@ class Command(BaseCommand):
 			).distinct():
 				publications_relevant.add(publication.cite_key)
 
+		self.cache_path = Path('.choices.semanticscholar.pickle')
+		if self.cache_path.exists():
+			self.echo("Loading choices from snowballing...")
+			with self.cache_path.open('rb') as f:
+				self.cache = pickle.load(f)
+				self.echo(f"{len(self.cache)} publications excluded during snowballing")
+				self.echo(f"{len(Publication.objects.filter(variant_of__isnull=True))} publications included in total so far")
+
 		# Output
 		self.echo(f"Total publications: {len(publications_found):4d}", bold=True)
 		self.echo(f"- peer reviewed:    {len(publications_peer_reviewed):4d}", bold=True)
 		self.echo(f"- relevant:         {len(publications_relevant):4d}", bold=True)
+
+		pubs = Publication.objects.values('title').annotate(count=Count('referenced_by')).values('title', 'year', 'count').order_by('count').reverse()
+		for p in pubs[:50]: print(f"{p['title']} ({p['year']}): {p['count']} citations")
+
+		years = Publication.objects.values('year').annotate(count=Count('year')).values('year', 'count')
+		for y in years: print(f"{y['year']}, {y['count']}")
+

sok/management/commands/zimport.py

@@ -0,0 +1,154 @@
+import html
+import io
+import string
+import csv
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+
+import requests
+
+from django.db import transaction
+from django.core.management.base import BaseCommand, CommandParser, CommandError
+
+from sok.models import (
+	Author,
+	Publication,
+	PublicationAuthor,
+	PublicationSource,
+	PublicationTag,
+	SearchTerm,
+	Source,
+	Tag,
+)
+
+
+
+# TODO: anpassen für WoS?
+PUBLICATIONS = {
+	'article',
+	'inproceedings',
+	'proceedings',
+	'book',
+	'incollection',
+	'phdthesis',
+	'mastersthesis',
+	'www',
+	'person',
+	'data',
+}
+
+CITE_KEY_PREFIX = 'Z:'
+
+
+class Command(BaseCommand):
+
+	def log_success(self, msg: str):
+		self.stdout.write(self.style.SUCCESS(msg))
+
+	def log_info(self, msg: str, nl: bool = True):
+		self.stdout.write(self.style.HTTP_INFO(msg), ending='\n' if nl else '')
+		self.stdout.flush()
+
+	# BaseCommand
+
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('--search-term', default='Not specified')
+		parser.add_argument('--source', default='Zotero')
+		parser.add_argument('zfile')
+
+	@transaction.atomic
+	def handle(self, *args, **options):
+		source, created = Source.objects.get_or_create(name=options['source'])
+
+		search_term: Optional[SearchTerm] = None
+		if name := options['search_term']:
+			search_term, created = SearchTerm.objects.get_or_create(name=name)
+			if created:
+				self.log_success(f"Created search term: {search_term}")
+
+		publications: List[Publication] = []
+
+		zotero_file = options['zfile']
+		with open(zotero_file, 'r') as csvfile:
+			reader = csv.DictReader(csvfile)
+			for publ in reader:
+
+				authors: List[Author] = []
+				for name in set(publ['Author'].split('; ')):
+					author, created = Author.objects.get_or_create(name=name)
+					if created:
+						self.log_success(f"Added author: {author}")
+					else:
+						self.log_info(f"Author '{author}' already known")
+					authors.append(author)
+
+				tags: List[Tag] = []
+				for t in set(publ['Manual Tags'].split('; ')).union(publ['Automatic Tags'].split('; ')):
+					if t == '': continue
+					tag, created = Tag.objects.get_or_create(name=t)
+					if created:
+						self.log_success(f"Added tag: {tag}")
+					else:
+						self.log_info(f"Tag '{tag}' already exists")
+					tags.append(tag)
+
+				pages = (None, None)
+				if '-' in publ['Pages']: 
+					pages = publ['Pages'].split('-')
+
+				# Add publication to database
+				publication, created = Publication.objects.get_or_create(
+					cite_key=publ['Key'],
+					title=publ['Title'],
+					year=publ['Publication Year'],
+					# TODO: peer_reviewed=result.is_peer_reviewed,
+					first_page=pages[0],
+					last_page=pages[1],
+					doi=publ['DOI'] or None,
+					abstract=publ['Abstract Note'] or None,
+				)
+				if created:
+					self.log_success(f"Added publication: {publication}")
+				else:
+					self.log_info(f"Publication '{publication}' already known")
+				publications.append(publication)
+
+				# Assign authors
+				for position, author in enumerate(authors):
+					publication_author, created = PublicationAuthor.objects.get_or_create(
+						author=author,
+						publication=publication,
+						position=position,
+					)
+					if created:
+						self.log_success(f"Assigned author '{author}' to publication '{publication}' at position {position}")
+					else:
+						self.log_info(f"Author '{author}' already assigned to publication '{publication}' at position '{position}'")
+
+				# Assign tags
+				for position, tag in enumerate(tags):
+					publication_tag, created = PublicationTag.objects.get_or_create(
+						tag=tag,
+						publication=publication,
+					)
+					if created:
+						self.log_success(f"Assigned tag '{tag}' to publication '{publication}'")
+					else:
+						self.log_info(f"Tag '{tag}' already assigned to publication '{publication}'")
+
+		# Assign sources
+		if search_term is not None:
+			for publication in publications:
+				print(publication, search_term, source)
+				publication_source, created = PublicationSource.objects.get_or_create(
+					source=source,
+					publication=publication,
+					search_term=search_term,
+				)
+				if created:
+					self.log_success(f"Assigned source '{source}' to publication '{publication}' with search term '{search_term}'")
+				else:
+					self.log_info(f"Source '{source}' already assigned to publication '{publication}' with search term '{search_term}'")

sok/migrations/0007_author_dblp_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.1 on 2025-05-16 08:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sok', '0006_alter_publication_abstract'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='author',
+            name='dblp_id',
+            field=models.CharField(blank=True, max_length=127),
+        ),
+    ]

sok/models.py

@@ -7,6 +7,7 @@ from django.db.models.query import QuerySet
 
 class Author(models.Model):
 	name = models.CharField(max_length=255, unique=True)
+	dblp_id = models.CharField(max_length=127, blank=True)
 
 	def __str__(self) -> str:
 		return self.name
@@ -103,6 +104,24 @@ class Publication(models.Model):
 	def relevant_referenced_by(self) -> QuerySet:
 		return self.referenced_by.filter(exclusion_criteria__isnull=True)
 
+	def stage_added(self) -> int:
+		if not self.is_relevant:
+			return 10000
+
+		if self.sources.exists():
+			return 0
+
+		if self.referenced_by.filter(exclusion_criteria__isnull=True, sources__isnull=False) or  self.references.filter(exclusion_criteria__isnull=True, sources__isnull=False):
+			return 1
+
+		if ref_by := self.referenced_by.filter(exclusion_criteria__isnull=True):
+			stages = set([ref.stage_added() for ref in ref_by])
+			return 1 + sorted(stages)[0]
+
+		if refs := self.references.filter(exclusion_criteria__isnull=True):
+			stages = set([ref.stage_added() for ref in refs])
+			return 1 + sorted(stages)[0]
+
 	@property
 	def stage(self) -> Optional[str]:
 		if not self.is_relevant:
@@ -113,24 +132,27 @@ class Publication(models.Model):
 			return 'primary'
 
 		# Referenced by primary (backward snowballing)
-		# TODO make transitive
+		# DONE make transitive
 		if self.referenced_by.filter(exclusion_criteria__isnull=True, sources__isnull=False):
 			return 'secondary'
 
 		# References a primary (forward snowballing)
-		# TODO make transitive
 		if self.references.filter(exclusion_criteria__isnull=True, sources__isnull=False):
 			return 'tertiary'
 
-		if ref_by := self.referenced_by.filter(exclusion_criteria__isnull=True):
-			stages = set([ref.stage for ref in ref_by])
-			if 'secondary' in stages: return 'sec-secondary'
-			elif 'tertiary' in stages: return 'sec-tertiary'
+		# prefer secondary over tertiary regardless of stage for now
+		exclude = False
+		if self.referenced_by.filter(exclusion_criteria__isnull=True):
+			stage = self.stage_added()
+			if stage > 10000: exclude = True
+			else: return str(stage) + '-secondary'
 
-		if refs := self.references.filter(exclusion_criteria__isnull=True):
-			stages = set([ref.stage for ref in ref_by])
-			if 'secondary' in stages: return 'tert-secondary'
-			elif 'tertiary' in stages: return 'tert-tertiary'
+		if self.references.filter(exclusion_criteria__isnull=True):
+			stage = self.stage_added()
+			if stage > 10000: return 'excluded-by-ref'
+			else: return str(stage) + '-tertiary'
+
+		if exclude: return 'excluded-by-ref'
 
 		return None
 

sok/static/tag/style.css

@@ -0,0 +1,3 @@
+li a {
+    color: green;
+}

sok/templates/tag/detail.html

@@ -0,0 +1,21 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8" />
+    <title>Tagging publications</title>
+  </head>
+  <body>
+    {% if publication %}
+        {% for author in publication.authors.all %}
+            <a>{{ author.name }}, </a>
+        {% endfor %}
+        <h2>{{ publication.title }} ({{ publication.year }})</h2>
+        {% if publication.doi %}
+        <!-- atm this is a link under /tag as well! -->
+            <a href="https://doi.org/{{ publication.doi }}">{{ publication.doi }}</a>
+        {% endif %}
+    {% else %}
+        <p>No publications are available.</p>
+    {% endif %}
+  </body>
+</html>

sok/templates/tag/index.html

@@ -0,0 +1,21 @@
+<!doctype html>
+{% load static %}
+
+<link rel="stylesheet" href="{% static 'tag/style.css' %}">
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8" />
+    <title>List for tagging publications</title>
+  </head>
+  <body>
+    {% if newest_publications %}
+        <ul>
+        {% for publication in newest_publications %}
+        <li><a href="{% url 'detail' publication.cite_key %}/">{{ publication.title }}, {{publication.year}}</a></li>
+        {% endfor %}
+        </ul>
+    {% else %}
+        <p>No publications are available.</p>
+    {% endif %}
+  </body>
+</html>

sok/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+    path("", views.IndexView.as_view(), name="index"),
+	path("<path:cite_key>", views.detail, name="detail"),
+]

sok/views.py

@@ -1,3 +1,31 @@
-from django.shortcuts import render
+from django.db.models import F
+from django.http import HttpResponseRedirect
+from django.shortcuts import get_object_or_404, render
+from django.urls import reverse
+from django.views import generic
+
+from sok.models import Publication, PublicationReference
+
+
+class IndexView(generic.ListView):
+	template_name = "tag/index.html"
+	context_object_name = "newest_publications"
+
+	def get_queryset(self):
+		return Publication.objects.order_by("-year")[:10]
+
+def detail(request, cite_key):
+	try:
+		publication = get_object_or_404(Publication, cite_key=cite_key.strip('/'))
+	except Publication.DoesNotExist:
+		raise Http404("Publication does not exist")
+	return render(request, "tag/detail.html", {"publication": publication})
+
+
+# def overview(request):
+# 	try:
+#         ps = Publication.objects.get(...)
+#     except Publication.DoesNotExist:
+#         raise Http404("Publication does not exist")
+#     return render(request, "publications/overview.html", {"to_tag": ps})
 
-# Create your views here.

sokman/settings.py

@@ -32,9 +32,17 @@ def get_or_generate_key() -> str:
 	assert path.exists()
 	return path.read_text()
 
+def get_api_key() -> str:
+	path = Path(Path(__file__).parent,'api.secret')
+
+	assert path.exists()
+	return path.read_text()
+
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = get_or_generate_key()
 
+SCHOLAR_API_KEY = get_api_key().strip('\n')
+
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
@@ -89,9 +97,13 @@ WSGI_APPLICATION = 'sokman.wsgi.application'
 
 DATABASES = {
 	'default': {
+		'ENGINE': 'django.db.backends.sqlite3',
+		'NAME': BASE_DIR / 'jens.sqlite3',
+	},
+	'uc_sok': {
 		'ENGINE': 'django.db.backends.sqlite3',
 		'NAME': BASE_DIR / 'db.sqlite3',
-	}
+	},
 }
 
 

sokman/urls.py

@@ -14,8 +14,9 @@ Including another URLconf
 	2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path
+from django.urls import path, include
 
 urlpatterns = [
 	path('admin/', admin.site.urls),
+	path('tag/', include('sok.urls')),
 ]