Merge pull request #445 from arXiv/develop

deploys funny paper id changes

browse/controllers/abs_page.py

@@ -106,6 +106,9 @@ def get_abs_page(arxiv_id: str) -> Response:
     response_data: Dict[str, Any] = {}
     response_headers: Dict[str, Any] = {}
     try:
+        if not Identifier.is_mostly_safe(arxiv_id):
+            raise AbsNotFound(data={"reason": "poorly formatted paper id"})
+
         arxiv_id = _check_legacy_id_params(arxiv_id)
         arxiv_identifier = Identifier(arxiv_id=arxiv_id)
 

browse/controllers/year.py

@@ -75,7 +75,7 @@ def year_page(archive_id: str, year: Optional[int]) -> Any:
             yymm= f"{str(month_count.year)[2:]}{month_count.month:02}",
             my = date(year=int(month_count.year),month=int(month_count.month), day=1).strftime("%b %Y"),
             url= url_for('browse.list_articles', context=archive_id,
-                         subcontext=f"{month_count.year}{month_count.month:02}"))
+                         subcontext=f"{month_count.year%100:02}{month_count.month:02}")) #TODO return to 4 digit year when all of listings is running on browse
         for month_count in count_listing.by_month]
 
     response_data: Dict[str, Any] = {
@@ -101,7 +101,7 @@ def year_page(archive_id: str, year: Optional[int]) -> Any:
 def ascii_art_month(archive_id: str, month: MonthCount) -> List[Tuple[str, Optional[str]]]:
     """Make ascii art for a MonthTotal."""
     tot = month.new + month.cross
-    yyyymm = f"{month.year}{month.month:02}"
+    yyyymm = f"{month.year%100:02}{month.month:02}" #TODO return to 4 digit year when all of listings is running on browse
 
     def _makestep(idx: int) -> Tuple[str, Optional[str]]:
         if idx % ASCII_ART_URL_STEP == 0:

browse/domain/identifier.py

@@ -209,3 +209,14 @@ def __eq__(self, other: object) -> bool:
             return self.__dict__ == other.__dict__
         except AttributeError:
             return False
+
+
+    @staticmethod
+    def is_mostly_safe(idin: Optional[str]) -> bool:
+        """Checks that the input could reasonably be parsed as an ID,
+        fails if strange unicode, starts with strange characters, very long etc."""
+        if not idin:
+            return False
+        if len(idin) > 200:
+            return False
+        return bool(re.match(re.compile(r"^[./0-9a-zA-Z-]{8}"), idin))

browse/services/database/__init__.py

@@ -478,7 +478,7 @@ def get_yearly_article_counts(archive: str, year: int) -> YearCount:
     """fetch total of new and cross-listed articles by month for a given category and year
         supports both styles of ids at once
     """
-    if "." not in archive: #adds ending to archive name requests
+    if archive=="math" and "." not in archive: #seperates math-ph from the general math category
         archive=archive+"."
 
     #filters to the correct database query based on the year the id schema changed

tests/conftest.py

@@ -21,12 +21,18 @@
 
 from tests import path_of_for_test
 
+import browse.services.documents as documents
+from browse.services.listing import db_listing
+
 DEFAULT_DB = "sqlite:///../tests/data/browse.db"
 TESTING_LATEXML_DB = 'sqlite:///../tests/data/latexmldb.db'
 
 
 TESTING_CONFIG = {
     "SQLALCHEMY_BINDS": {"latexml": TESTING_LATEXML_DB},
+    "SQLALCHEMY_DATABASE_URI" : DEFAULT_DB,
+    'DOCUMENT_LISTING_SERVICE': db_listing,
+    'DOCUMENT_ABSTRACT_SERVICE': documents.db_docs,
     "APPLICATION_ROOT": "",
     "TESTING": True,
     }
@@ -37,23 +43,18 @@ def test_config():
 @pytest.fixture(scope='session')
 def loaded_db():
     """Loads the testing db"""
-    app = create_web_app()
+    app = create_web_app(**test_config())
     with app.app_context():
         from browse.services.database import models
-
         from . import populate_test_database
         populate_test_database(True, models)
 
 
 @pytest.fixture(scope='session')
 def app_with_db(loaded_db):
     """App setup with DB backends."""
-    import browse.services.documents as documents
-    from browse.services.listing import db_listing
 
     conf = test_config()
-    conf.update({'DOCUMENT_LISTING_SERVICE': db_listing})
-    conf.update({'DOCUMENT_ABSTRACT_SERVICE': documents.db_docs})
     app = create_web_app(**conf)
 
     with app.app_context():

tests/test_browse.py

@@ -652,3 +652,10 @@ def test_withdrawn_then_new_version(self):
 
         self.assertIn("Older arxiv papers may lack submitter name", txt,
                       "Expect a message about lack of submitter name on old paper")
+
+
+    def test_bad_data(self):
+        """Test where v2 is withdrawn but then there is a non-withdrawn v3."""
+        rv = self.client.get('/abs/gr-qc/۹۷۰۶123')
+        self.assertNotEqual(rv.status_code, 200)
+        self.assertNotEqual(rv.status_code, 500)

tests/test_hybrid_listing.py

@@ -151,9 +151,9 @@ def test_year_page_hybrid(mock, client_with_hybrid_listings):
     assert '<a href="/year/cond-mat/11">2011</a>' in text
     assert "<p>2009 totals: <b>8 articles</b> + <i>2 cross-lists</i></p>" in text
     assert (
-        "<a href=/list/cond-mat/200911?skip=0>|</a>      <b>2</b> + 0 (Nov 2009)"
+        "<a href=/list/cond-mat/0911?skip=0>|</a>      <b>2</b> + 0 (Nov 2009)"
         in text
-    )
+    ) #TODO change this back to 4 digit year when all of listings is running on browse
     assert '<a href="/year/cond-mat/92">1992</a>' in text
 
     rv = client.get("/year/cs/23")
@@ -186,3 +186,27 @@ def test_monthly_counts_hybrid(app_with_hybrid_listings):
         ]
         year = YearCount(2009, 8, 2, months)
         assert result == year
+
+def test_finds_archives_with_no_categories(app_with_hybrid_listings):
+    app = app_with_hybrid_listings
+    with app.app_context():
+
+        months = [
+            MonthCount(2009, 1, 0, 0),
+            MonthCount(2009, 2, 0, 0),
+            MonthCount(2009, 3, 0, 0),
+            MonthCount(2009, 4, 0, 0),
+            MonthCount(2009, 5, 0, 0),
+            MonthCount(2009, 6, 1, 0),
+            MonthCount(2009, 7, 1, 0),
+            MonthCount(2009, 8, 0, 0),
+            MonthCount(2009, 9, 0, 0),
+            MonthCount(2009, 10, 0, 0),
+            MonthCount(2009, 11, 0, 0),
+            MonthCount(2009, 12, 0, 0),
+        ]
+        year1 = YearCount(2009, 2, 0, months)
+
+        assert year1 == get_yearly_article_counts(
+            "gr-qc", 2009
+        )