I don't know but this is the current state

sok/management/commands/dblpimport.py

@@ -96,7 +96,8 @@ class PublicationResult:
 	year: int
 	pages: Optional[Tuple[int, int]]
 	dblp_doi: Optional[str] = None
-	authors: List[str] = field(default_factory=list)
+	access: Optional[str] = None
+	authors: List[Tuple[str,str]] = field(default_factory=list)
 	urls: List[str] = field(default_factory=list)
 
 	@property
@@ -166,7 +167,7 @@ class PublicationResult:
 
 		url = f"https://dblp.uni-trier.de/rec/{key}.xml"
 		response = requests.get(url)
-		response.raise_for_status
+		response.raise_for_status()
 
 		parser = xml.sax.make_parser()
 		handler = DBLPHandler({key})
@@ -195,6 +196,7 @@ class PublicationResult:
 
 		# TODO Parse URLs ('ee')
 		doi = None if not 'doi' in info else info['doi']
+		access = None if not 'access' in info else info['access']
 
 		return cls(
 			key=info['key'],
@@ -202,7 +204,8 @@ class PublicationResult:
 			year=int(info['year']),
 			pages=pages,
 			dblp_doi=doi,
-			authors=[html.unescape(author['text']) for author in authors],
+			access = access,
+			authors=[(author['@pid'], html.unescape(author['text'])) for author in authors],
 		)
 
 	@classmethod
@@ -223,10 +226,10 @@ class PublicationResult:
 				'format': 'json',
 			},
 		)
-		response.raise_for_status
+		response.raise_for_status()
 		search_result = response.json()['result']
 		hits = search_result['hits']
-		if hits['@total'] == '0': return (None, None, 0)
+		if hits['@total'] == '0' or hits['@sent'] == '0': return (None, None, 0)
 		results = [cls.from_search_hit(hit) for hit in hits['hit']]
 
 		total = hits['@total']
@@ -453,6 +456,7 @@ class Command(BaseCommand):
 				# Add authors to database
 				authors: List[Author] = []
 				for name in result.authors:
+					# TODO? find author id's -> not in xml!
 					author, created = Author.objects.get_or_create(name=name)
 					if created:
 						self.log_success(f"Added author: {author}")

sok/management/commands/dblpsearch.py

@@ -34,7 +34,7 @@ class Command(BaseCommand):
 		self.stdout.write("")
 		self.log_info(result.cite_key)
 		if 0 < len(result.authors):
-			self.stdout.write("  " + ", ".join([name for name in result.authors]))
+			self.stdout.write("  " + ", ".join([author.name for author in result.authors]))
 		self.log_info("  " + result.title, nl=False)
 		self.stdout.write(f" ({result.year})")
 
@@ -66,8 +66,16 @@ class Command(BaseCommand):
 
 		# Store Authors
 		authors: List[Author] = []
-		for name in result.authors:
+		for (pid, name) in result.authors:
 			author, created = Author.objects.get_or_create(name=name)
+			if not author.dblp_id and pid != None:
+				try:
+					author.dblp_id = pid
+					author.full_clean()
+					author.save()
+				except ValidationError as e:
+					raise CommandError(f"{author}: {e}")
+
 			if created:
 				self.log_success(f"Added author: {author}")
 			else:

sok/management/commands/repair.py

@@ -2,14 +2,19 @@ from pprint import pprint
 from time import sleep
 from tqdm import tqdm
 
+import re, requests, html
+
 from django.db import transaction
 from django.core.exceptions import ValidationError
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+from django.db.models import Count
+
+from typing import List, Optional, Tuple
 
 import sok.management.commands.dblpimport as dblp
 
 from sok.management.commands.snowball import semanticscholar
-from sok.models import Publication, PublicationReference, SemanticScholar
+from sok.models import Publication, PublicationReference, SemanticScholar, Author, PublicationAuthor
 
 
 class Command(BaseCommand):
@@ -17,6 +22,9 @@ class Command(BaseCommand):
 	def log_success(self, msg: str):
 		self.stdout.write(self.style.SUCCESS(msg))
 
+	def log_warn(self, msg: str):
+		self.stdout.write(self.style.WARNING(msg))
+
 	def log_info(self, msg: str, nl: bool = True):
 		self.stdout.write(self.style.HTTP_INFO(msg), ending='\n' if nl else '')
 		self.stdout.flush()
@@ -71,6 +79,7 @@ class Command(BaseCommand):
 				data = semanticscholar(semantic.paper_id)
 				if abstract := data['abstract']:
 					publication.abstract = abstract
+					if publication.peer_reviewed == None: publication.peer_reviewed = False
 					try:
 						publication.full_clean()
 						publication.save()
@@ -83,30 +92,79 @@ class Command(BaseCommand):
 	def search_on_dblp(self, publication: Publication):
 		query, results, total = dblp.PublicationResult.from_search(publication.title, 100)
 		if total == 0: return
+		plausible = []
 		for result in results:
 			if publication.doi and result.doi:
 				if publication.doi.lower() == result.doi.lower():
-					publication.cite_key = result.cite_key
+					plausible = [result]
-					publication.year = result.year
+					break
-					if not result.is_peer_reviewed == None:
-						publication.peer_reviewed = result.is_peer_reviewed
-					publication.first_page = result.first_page
-					publication.last_page = result.last_page
-					print(publication.peer_reviewed, result)
-					# keep authors from semantic scholar for now, even though they might be a little broken?
-					try:
-						publication.full_clean()
-						publication.save()
-						self.log_success(f"Added DBLP info for: {publication}")
-					except ValidationError as e:
-						raise CommandError(f"{publication}: {e}")
 				else: continue # quite definitely not the same publication
 			elif publication.title == result.title and publication.year == result.year:
-				print(f"Not quite certain about {result.cite_key} for publication {publication.title}")
+				plausible.append(result)
+				self.log_warn(f"Not quite certain about {result.cite_key} for publication {publication.title}")
 			else: continue # I'd rather look at that in detail for now
+			if len(plausible) == 1:
+				result = plausible[0]
+				if result.access.lower() == 'withdrawn':
+					self.log_warn(f"Publication {publication.title} was WITHDRAWN!")
+					return
+				publication.cite_key = result.cite_key
+				publication.year = result.year
+				if not result.is_peer_reviewed == None:
+					publication.peer_reviewed = result.is_peer_reviewed
+				publication.first_page = result.first_page
+				publication.last_page = result.last_page
+				print(publication.peer_reviewed, result)
+				try:
+					publication.full_clean()
+					publication.save()
+					self.log_success(f"Added DBLP info for: {publication}")
+				except ValidationError as e:
+					raise CommandError(f"{publication}: {e}")
+
+				# Store Authors
+				authors: List[Author] = []
+				for (pid, name) in result.authors:
+					author, created = Author.objects.get_or_create(name=name)
+
+					if created:
+						self.log_success(f"Added author: {author}")
+					else:
+						self.log_info(f"Author '{author}' already known")
+					if not author.dblp_id and pid != None:
+						try:
+							author.dblp_id = pid
+							author.full_clean()
+							author.save()
+							self.log_success(f"Added pid to author: {author}")
+						except ValidationError as e:
+							raise CommandError(f"{author}: {e}")
+
+					authors.append(author)
+
+				# Replace authors for publication
+				if len(authors) != len(PublicationAuthor.objects.filter(publication=publication)): print(authors, [pa.author for pa in PublicationAuthor.objects.filter(publication=publication)])
+				assert len(authors) >= len(PublicationAuthor.objects.filter(publication=publication))
+				for publication_author in PublicationAuthor.objects.filter(publication=publication):
+					self.log_warn(f"Will now try to delete {publication_author.author} from {publication}")
+					try:
+						publication_author.delete()
+					except e:
+						raise CommandError(f"{publication} - {author}: {e}")
+
+				for position, author in enumerate(authors):
+					publication_author, created = PublicationAuthor.objects.get_or_create(
+						author=author,
+						publication=publication,
+						position=position,
+					)
+					if created:
+						self.log_success(f"Assigned author '{author}' to publication '{publication}' at position {position}")
+					else:
+						self.log_info(f"Author '{author}' already assigned to publication '{publication}' at position '{position}'")
 
 	def find_secondary_on_dblp(self):
-		self.log_info("--- Searching for secondary and tertiary sources in the default DBLP dump ---")
+		self.log_info("--- Searching for snowballed sources on DBLP ---")
 		for publication in tqdm(Publication.objects.exclude(cite_key__startswith=dblp.CITE_KEY_PREFIX), unit="publication"):
 			if publication.stage == 'excluded': continue
 			self.search_on_dblp(publication)
@@ -117,7 +175,7 @@ class Command(BaseCommand):
 			).values_list('cite_key', flat=True).distinct()
 		)
 		n = len(keys)
-		self.log_info(f"Found {n} publications that still need to be checked")
+		self.log_info(f"Found {n} publications that still need to be verified")
 
 	def fix_dblp(self):
 		self.log_info("--- Searching for entries not in the default DBLP dump ---")
@@ -169,12 +227,149 @@ class Command(BaseCommand):
 
 			sleep(2)  # Throttle to avoid rate-limiting
 
+	def find_author_on_dblp(
+		self,
+		name: str,
+		limit: int = 100,
+		) -> List[Tuple[str, str]]:
+		url = 'http://dblp.org/search/author/api'
+		response = requests.get(
+			url,
+			params={
+				'q': name,
+				'f': 0,
+				'h': limit,
+				'c': 0,
+				'format': 'json',
+			},
+		)
+		response.raise_for_status()
+		search_result = response.json()['result']
+		hits = search_result['hits']
+		if hits['@total'] == '0': return None
+		results = [hit['info'] for hit in hits['hit']]
+
+		return results
+
+	def verify_author(self, pid, keys):
+		url = f"https://dblp.org/pid/{pid}.xml"
+		response = requests.get(url)
+		response.raise_for_status()
+		pubs = re.findall('key="([^"]*)"', str(response.content))
+		sleep(1)  # Throttle to avoid rate-limiting
+		return keys.intersection(pubs) == keys
+
+	def find_pid_for_authors(self):
+		self.log_info("--- Adding ID's to authors ---")
+		authors = Author.objects.filter(dblp_id__exact="")
+		for author in tqdm(authors, unit='author'):
+			pid = ""
+			keys = set({
+				dblp.strip_cite_key_prefix(cite_key)
+				for cite_key in PublicationAuthor.objects.filter(
+					author__exact=author,
+					publication__cite_key__startswith=dblp.CITE_KEY_PREFIX
+				).values_list('publication__cite_key', flat=True).distinct()}
+			)
+			if len(keys) == 0: continue
+			options = self.find_author_on_dblp(author.name)
+			if not options:
+				self.log_warn(f"Could not find ID for author {author}")
+			elif len(options) == 1:
+				pid = re.match('https://dblp.org/pid/(.*)', options[0]['url']).groups()[0]
+				if self.verify_author(pid, keys):
+					self.log_success(f"Found {pid} for author {author}")
+					author.dblp_id = pid
+					author.full_clean()
+					author.save()
+					self.log_success(f"Added pid for author {author}")
+			else:
+				for opt in options:
+					if not 'url' in opt: continue
+					pid = re.match('https://dblp.org/pid/(.*)', opt['url']).groups()[0]
+					if self.verify_author(pid, keys):
+						self.log_success(f"Found {pid} for author {author}")
+						author.dblp_id = pid
+						author.full_clean()
+						author.save()
+						self.log_success(f"Added pid for author {author}")
+						break
+
+	@transaction.atomic
+	def merge(self, author, variant):
+		for pub_author in PublicationAuthor.objects.filter(author=variant):
+			publication = pub_author.publication
+			if PublicationAuthor.objects.filter(author=author, publication=publication).exists():
+				self.log_warn(f"{author} is already author of {publication}")
+				break
+			fixed = PublicationAuthor(
+				author=author,
+				publication=publication,
+				position=pub_author.position
+			)
+			try:
+				pub_author.delete()
+				fixed.full_clean()
+				fixed.save()
+				self.log_success(f"Changed author: {publication}: {variant} -> {fixed.author}")
+			except ValidationError as e:
+				raise CommandError(f"{publication}: {variant} -> {fixed.author}: {e}")
+		assert not PublicationAuthor.objects.filter(author=variant).exists()
+		variant.delete()
+
+
+
+	def merge_authors(self) -> None:
+		self.log_info("--- Searching for potential duplicate authors and merging their publications ---")
+
+		dup_ids = Author.objects.exclude(dblp_id__exact="").values_list('dblp_id').annotate(count=Count('id')).filter(count__gt=1)
+
+		try:
+			for (pid, counter) in tqdm(dup_ids, unit="duplicate"):
+
+				response = requests.get(f"https://dblp.org/pid/{pid}.xml")
+				response.raise_for_status()
+				dblp_person = re.search('dblpperson name="([^"]*)"', response.content.decode('ascii')).groups()
+				assert len(dblp_person) == 1
+				dblp_name = html.unescape(dblp_person[0])
+
+				variants = Author.objects.filter(dblp_id__exact=pid).exclude(name__exact=dblp_name)
+				# for v in variants:
+				# 	print(v.name, v.dblp_id)
+				# break
+				orig = Author.objects.filter(dblp_id__exact=pid, name__exact=dblp_name)
+				print(variants, dblp_name, orig)
+				assert len(orig) == 1
+
+				self.log_info(f"Suggestion to merge {', '.join([v.name for v in variants])} into {orig[0]}")
+				
+				while True:
+					self.log_warn("Merge? [Y/n]")
+					choice = input().lower()
+					if choice in {'', 'y', 'yes'}:
+						for variant in variants: self.merge(orig[0], variant)
+						break
+					elif choice in {'', 'n', 'no'}:
+						break
+		except KeyboardInterrupt:
+			raise CommandError("Aborted.")
+
 	# BaseCommand
+	def add_arguments(self, parser: CommandParser):
+		parser.add_argument('-b', '--abstract', action='store_true')
+		parser.add_argument('-d', '--dblp', action='store_true')
+		parser.add_argument('-a', '--authors', action='store_true')
+		parser.add_argument('-s', '--secondary', action='store_true')
 
 	def handle(self, *args, **options):
 		self.fix_references()
-		self.fix_abstracts()
+		if options['secondary']: self.find_secondary_on_dblp()
-		self.find_secondary_on_dblp()
+		if options['dblp']:
-		self.fix_dblp()
+			self.fix_dblp()
-		self.find_missing_dois()
+			self.find_missing_dois()
-		self.find_semanticscholar_ids()
+			self.find_semanticscholar_ids()
+		if options['authors']:
+			self.find_pid_for_authors()
+			self.merge_authors()
+		if options['abstract']: self.fix_abstracts()
+

sok/management/commands/stats.py

@@ -1,4 +1,8 @@
 from typing import Set
+import pickle
+from django.db.models import Count
+
+from pathlib import Path
 
 from django.core.management.base import BaseCommand
 from django.db.models import Count, Q
@@ -43,7 +47,22 @@ class Command(BaseCommand):
 			).distinct():
 				publications_relevant.add(publication.cite_key)
 
+		self.cache_path = Path('.choices.semanticscholar.pickle')
+		if self.cache_path.exists():
+			self.echo("Loading choices from snowballing...")
+			with self.cache_path.open('rb') as f:
+				self.cache = pickle.load(f)
+				self.echo(f"{len(self.cache)} publications excluded during snowballing")
+				self.echo(f"{len(Publication.objects.filter(variant_of__isnull=True))} publications included in total so far")
+
 		# Output
 		self.echo(f"Total publications: {len(publications_found):4d}", bold=True)
 		self.echo(f"- peer reviewed:    {len(publications_peer_reviewed):4d}", bold=True)
 		self.echo(f"- relevant:         {len(publications_relevant):4d}", bold=True)
+
+		pubs = Publication.objects.values('title').annotate(count=Count('referenced_by')).values('title', 'year', 'count').order_by('count').reverse()
+		for p in pubs[:50]: print(f"{p['title']} ({p['year']}): {p['count']} citations")
+
+		years = Publication.objects.values('year').annotate(count=Count('year')).values('year', 'count')
+		for y in years: print(f"{y['year']}, {y['count']}")
+

sok/migrations/0007_author_dblp_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.1 on 2025-05-16 08:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sok', '0006_alter_publication_abstract'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='author',
+            name='dblp_id',
+            field=models.CharField(blank=True, max_length=127),
+        ),
+    ]

sok/models.py

@@ -7,6 +7,7 @@ from django.db.models.query import QuerySet
 
 class Author(models.Model):
 	name = models.CharField(max_length=255, unique=True)
+	dblp_id = models.CharField(max_length=127, blank=True)
 
 	def __str__(self) -> str:
 		return self.name

sok/static/tag/style.css

@@ -0,0 +1,3 @@
+li a {
+    color: green;
+}

sok/templates/tag/detail.html

@@ -0,0 +1,21 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8" />
+    <title>Tagging publications</title>
+  </head>
+  <body>
+    {% if publication %}
+        {% for author in publication.authors.all %}
+            <a>{{ author.name }}, </a>
+        {% endfor %}
+        <h2>{{ publication.title }} ({{ publication.year }})</h2>
+        {% if publication.doi %}
+        <!-- atm this is a link under /tag as well! -->
+            <a href="https://doi.org/{{ publication.doi }}">{{ publication.doi }}</a>
+        {% endif %}
+    {% else %}
+        <p>No publications are available.</p>
+    {% endif %}
+  </body>
+</html>

sok/templates/tag/index.html

@@ -0,0 +1,21 @@
+<!doctype html>
+{% load static %}
+
+<link rel="stylesheet" href="{% static 'tag/style.css' %}">
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8" />
+    <title>List for tagging publications</title>
+  </head>
+  <body>
+    {% if newest_publications %}
+        <ul>
+        {% for publication in newest_publications %}
+        <li><a href="{% url 'detail' publication.cite_key %}/">{{ publication.title }}, {{publication.year}}</a></li>
+        {% endfor %}
+        </ul>
+    {% else %}
+        <p>No publications are available.</p>
+    {% endif %}
+  </body>
+</html>

sok/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+    path("", views.IndexView.as_view(), name="index"),
+	path("<path:cite_key>", views.detail, name="detail"),
+]

sok/views.py

@@ -1,3 +1,31 @@
-from django.shortcuts import render
+from django.db.models import F
+from django.http import HttpResponseRedirect
+from django.shortcuts import get_object_or_404, render
+from django.urls import reverse
+from django.views import generic
+
+from sok.models import Publication, PublicationReference
+
+
+class IndexView(generic.ListView):
+	template_name = "tag/index.html"
+	context_object_name = "newest_publications"
+
+	def get_queryset(self):
+		return Publication.objects.order_by("-year")[:10]
+
+def detail(request, cite_key):
+	try:
+		publication = get_object_or_404(Publication, cite_key=cite_key.strip('/'))
+	except Publication.DoesNotExist:
+		raise Http404("Publication does not exist")
+	return render(request, "tag/detail.html", {"publication": publication})
+
+
+# def overview(request):
+# 	try:
+#         ps = Publication.objects.get(...)
+#     except Publication.DoesNotExist:
+#         raise Http404("Publication does not exist")
+#     return render(request, "publications/overview.html", {"to_tag": ps})
 
-# Create your views here.

sokman/urls.py

@@ -14,8 +14,9 @@ Including another URLconf
 	2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path
+from django.urls import path, include
 
 urlpatterns = [
 	path('admin/', admin.site.urls),
+	path('tag/', include('sok.urls')),
 ]