Add some things
This commit is contained in:
Maya Herrscher committed
1 parent
3bbcc454c7
commit
fa4981693c
2 files changed
+82
-55
No files matched your search
@@ -92,68 +92,76 @@ class Command(BaseCommand):
|
||||
def search_on_dblp(self, publication: Publication):
|
||||
query, results, total = dblp.PublicationResult.from_search(publication.title, 100)
|
||||
if total == 0: return
|
||||
plausible = []
|
||||
for result in results:
|
||||
if publication.doi and result.doi:
|
||||
if publication.doi.lower() == result.doi.lower():
|
||||
publication.cite_key = result.cite_key
|
||||
publication.year = result.year
|
||||
if not result.is_peer_reviewed == None:
|
||||
publication.peer_reviewed = result.is_peer_reviewed
|
||||
publication.first_page = result.first_page
|
||||
publication.last_page = result.last_page
|
||||
print(publication.peer_reviewed, result)
|
||||
try:
|
||||
publication.full_clean()
|
||||
publication.save()
|
||||
self.log_success(f"Added DBLP info for: {publication}")
|
||||
except ValidationError as e:
|
||||
raise CommandError(f"{publication}: {e}")
|
||||
|
||||
# Store Authors
|
||||
authors: List[Author] = []
|
||||
for (pid, name) in result.authors:
|
||||
author, created = Author.objects.get_or_create(name=name)
|
||||
|
||||
if created:
|
||||
self.log_success(f"Added author: {author}")
|
||||
else:
|
||||
self.log_info(f"Author '{author}' already known")
|
||||
if not author.dblp_id and pid != None:
|
||||
try:
|
||||
author.dblp_id = pid
|
||||
author.full_clean()
|
||||
author.save()
|
||||
self.log_success(f"Added pid to author: {author}")
|
||||
except ValidationError as e:
|
||||
raise CommandError(f"{author}: {e}")
|
||||
|
||||
authors.append(author)
|
||||
|
||||
# Replace authors for publication
|
||||
if len(authors) != len(PublicationAuthor.objects.filter(publication=publication)): print(authors, [pa.author for pa in PublicationAuthor.objects.filter(publication=publication)])
|
||||
assert len(authors) >= len(PublicationAuthor.objects.filter(publication=publication))
|
||||
for publication_author in PublicationAuthor.objects.filter(publication=publication):
|
||||
self.log_warn(f"Will now try to delete {publication_author.author} from {publication}")
|
||||
try:
|
||||
publication_author.delete()
|
||||
except e:
|
||||
raise CommandError(f"{publication} - {author}: {e}")
|
||||
|
||||
for position, author in enumerate(authors):
|
||||
publication_author, created = PublicationAuthor.objects.get_or_create(
|
||||
author=author,
|
||||
publication=publication,
|
||||
position=position,
|
||||
)
|
||||
if created:
|
||||
self.log_success(f"Assigned author '{author}' to publication '{publication}' at position {position}")
|
||||
else:
|
||||
self.log_info(f"Author '{author}' already assigned to publication '{publication}' at position '{position}'")
|
||||
|
||||
plausible = [result]
|
||||
break
|
||||
else: continue # quite definitely not the same publication
|
||||
elif publication.title == result.title and publication.year == result.year:
|
||||
plausible.append(result)
|
||||
self.log_warn(f"Not quite certain about {result.cite_key} for publication {publication.title}")
|
||||
else: continue # I'd rather look at that in detail for now
|
||||
if len(plausible) == 1:
|
||||
result = plausible[0]
|
||||
if result.access.lower() == 'withdrawn':
|
||||
self.log_warn(f"Publication {publication.title} was WITHDRAWN!")
|
||||
return
|
||||
publication.cite_key = result.cite_key
|
||||
publication.year = result.year
|
||||
if not result.is_peer_reviewed == None:
|
||||
publication.peer_reviewed = result.is_peer_reviewed
|
||||
publication.first_page = result.first_page
|
||||
publication.last_page = result.last_page
|
||||
print(publication.peer_reviewed, result)
|
||||
try:
|
||||
publication.full_clean()
|
||||
publication.save()
|
||||
self.log_success(f"Added DBLP info for: {publication}")
|
||||
except ValidationError as e:
|
||||
raise CommandError(f"{publication}: {e}")
|
||||
|
||||
# Store Authors
|
||||
authors: List[Author] = []
|
||||
for (pid, name) in result.authors:
|
||||
author, created = Author.objects.get_or_create(name=name)
|
||||
|
||||
if created:
|
||||
self.log_success(f"Added author: {author}")
|
||||
else:
|
||||
self.log_info(f"Author '{author}' already known")
|
||||
if not author.dblp_id and pid != None:
|
||||
try:
|
||||
author.dblp_id = pid
|
||||
author.full_clean()
|
||||
author.save()
|
||||
self.log_success(f"Added pid to author: {author}")
|
||||
except ValidationError as e:
|
||||
raise CommandError(f"{author}: {e}")
|
||||
|
||||
authors.append(author)
|
||||
|
||||
# Replace authors for publication
|
||||
if len(authors) != len(PublicationAuthor.objects.filter(publication=publication)): print(authors, [pa.author for pa in PublicationAuthor.objects.filter(publication=publication)])
|
||||
assert len(authors) >= len(PublicationAuthor.objects.filter(publication=publication))
|
||||
for publication_author in PublicationAuthor.objects.filter(publication=publication):
|
||||
self.log_warn(f"Will now try to delete {publication_author.author} from {publication}")
|
||||
try:
|
||||
publication_author.delete()
|
||||
except e:
|
||||
raise CommandError(f"{publication} - {author}: {e}")
|
||||
|
||||
for position, author in enumerate(authors):
|
||||
publication_author, created = PublicationAuthor.objects.get_or_create(
|
||||
author=author,
|
||||
publication=publication,
|
||||
position=position,
|
||||
)
|
||||
if created:
|
||||
self.log_success(f"Assigned author '{author}' to publication '{publication}' at position {position}")
|
||||
else:
|
||||
self.log_info(f"Author '{author}' already assigned to publication '{publication}' at position '{position}'")
|
||||
|
||||
def find_secondary_on_dblp(self):
|
||||
self.log_info("--- Searching for snowballed sources on DBLP ---")
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
from typing import Set
|
||||
import pickle
|
||||
from django.db.models import Count
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db.models import Count, Q
|
||||
@@ -43,7 +47,22 @@ class Command(BaseCommand):
|
||||
).distinct():
|
||||
publications_relevant.add(publication.cite_key)
|
||||
|
||||
self.cache_path = Path('.choices.semanticscholar.pickle')
|
||||
if self.cache_path.exists():
|
||||
self.echo("Loading choices from snowballing...")
|
||||
with self.cache_path.open('rb') as f:
|
||||
self.cache = pickle.load(f)
|
||||
self.echo(f"{len(self.cache)} publications excluded during snowballing")
|
||||
self.echo(f"{len(Publication.objects.filter(variant_of__isnull=True))} publications included in total so far")
|
||||
|
||||
# Output
|
||||
self.echo(f"Total publications: {len(publications_found):4d}", bold=True)
|
||||
self.echo(f"- peer reviewed: {len(publications_peer_reviewed):4d}", bold=True)
|
||||
self.echo(f"- relevant: {len(publications_relevant):4d}", bold=True)
|
||||
|
||||
pubs = Publication.objects.values('title').annotate(count=Count('referenced_by')).values('title', 'year', 'count').order_by('count').reverse()
|
||||
for p in pubs[:15]: print(f"{p['title']} ({p['year']}): {p['count']} citations")
|
||||
|
||||
years = Publication.objects.values('year').annotate(count=Count('year')).values('year', 'count')
|
||||
for y in years: print(f"{y['year']}, {y['count']}")
|
||||
|
||||
Reference in new issue
Block a user