✅ Test storage_analyses utils, task

Author: znatty22

tests/files/fixtures.py

@@ -33,8 +33,12 @@ def _make(nrows=10, filename="upload_manifest.tsv"):
         version = VersionFactory()
         df = pandas.DataFrame(
             [
-                {c: c for c in MANIFEST_SCHEMA["required"]}
-                for i in range(nrows)
+                {
+                    "Source File Name": f"foo{i}.tsv",
+                    "Hash":  f"hash{i}",
+                    "Hash Algorithm": "MD5",
+                    "Size": i * 100,
+                } for i in range(nrows)
             ]
         )
         content = df.to_csv(sep="\t", index=False)

tests/storage_analyses/__init__.py

@@ -0,0 +1,140 @@
+import pytest
+import pandas
+
+from creator.files.models import AuditPrepState
+from creator.files.factories import VersionFactory
+from creator.studies.factories import StudyFactory
+from creator.storage_analyses.tasks import expected_file as task
+from creator.storage_analyses.factories import ExpectedFileFactory
+from creator.storage_analyses.models import AuditState, ExpectedFile
+
+from ..files.fixtures import make_file_upload_manifest
+
+
+def test_bulk_update_audit_state(db):
+    """
+    Test helper _bulk_update_audit_state
+    """
+    files = ExpectedFileFactory.create_batch(
+        5, audit_state=AuditState.NOT_SUBMITTED
+    )
+    task._bulk_update_audit_state(files, "start_submission")
+    for f in files:
+        assert f.audit_state == AuditState.SUBMITTING
+
+
+def test_dataframe_to_expected_files(db, mocker):
+    """
+    Test helper _dataframe_to_expected_files
+    """
+    study = StudyFactory()
+    mock_logger = mocker.patch(
+        "creator.storage_analyses.tasks.expected_file.logger"
+    )
+    rows = [
+        {
+            "Source File Name": f"foo{i}.tsv",
+            "Hash":  f"hash{i}",
+            "Hash Algorithm": "MD5",
+            "Size": i * 100,
+            "Foo": "Bar"
+        } for i in range(5)
+    ]
+    missing_req = {
+        "Hash": "1234",
+        "Hash Algorithm": "MD5"
+    }
+    df = pandas.DataFrame(rows + [rows[0].copy(), missing_req])
+    files = task._dataframe_to_expected_files(df, study)
+
+    assert len(files) == 5
+    assert files[0]["study_id"] == study.pk
+    assert files[0]["custom_fields"] == {"Foo": "Bar"}
+    assert "source_file_name" in files[0]
+    assert mock_logger.warning.call_count == 2
+
+
+def test_prepare_audit_submission(db, mocker, make_file_upload_manifest):
+    """
+    Test task to prepare audit submission
+    """
+    mock_django_rq = mocker.patch(
+        "creator.storage_analyses.tasks.expected_file.django_rq"
+    )
+    # Success
+    version = make_file_upload_manifest(nrows=10)
+    version.start_audit_prep()
+    version.save()
+    task.prepare_audit_submission(version.pk)
+    version.refresh_from_db()
+    assert mock_django_rq.enqueue.call_count == 1
+    assert version.audit_prep_state == AuditPrepState.COMPLETED
+    assert ExpectedFile.objects.count() == 10
+
+    # Failure
+    version = VersionFactory()
+    version.start_audit_prep()
+    version.save()
+    with pytest.raises(Exception):
+        task.prepare_audit_submission(version.pk)
+    version.refresh_from_db()
+    assert version.audit_prep_state == AuditPrepState.FAILED
+
+
+def test_submit_study_for_audit(db, mocker):
+    """
+    Test task to submit study for audit
+    """
+    mock_logger = mocker.patch(
+        "creator.storage_analyses.tasks.expected_file.logger"
+    )
+    mock_dewrangle = mocker.patch(
+        "creator.storage_analyses.tasks.expected_file.DewrangleClient"
+    )
+    mock_client = mock_dewrangle()
+    study = StudyFactory()
+    ExpectedFileFactory.create_batch(
+        10, audit_state=AuditState.NOT_SUBMITTED, study=study
+    )
+    ExpectedFileFactory.create_batch(
+        8, audit_state=AuditState.SUBMITTING, study=study
+    )
+    ExpectedFileFactory.create_batch(
+        5, audit_state=AuditState.FAILED, study=study
+    )
+    task.submit_study_for_audit(study.pk)
+    assert ExpectedFile.objects.filter(
+        audit_state=AuditState.SUBMITTED
+    ).count() == 15
+    assert ExpectedFile.objects.filter(
+        audit_state=AuditState.SUBMITTING
+    ).count() == 8
+    assert mock_client.upsert_organization.call_count == 1
+    assert mock_client.upsert_study.call_count == 1
+    assert mock_client.bulk_upsert_expected_files.call_count == 1
+
+
+def test_submit_study_for_audit_errors(db, mocker):
+    """
+    Test task to submit study for audit error cases
+    """
+    mock_logger = mocker.patch(
+        "creator.storage_analyses.tasks.expected_file.logger"
+    )
+    mock_dewrangle = mocker.patch(
+        "creator.storage_analyses.tasks.expected_file.DewrangleClient"
+    )
+    mock_client = mock_dewrangle()
+    study = StudyFactory()
+
+    # No expected files to submit
+    task.submit_study_for_audit(study.pk)
+    assert mock_logger.info.call_count == 1
+
+    # Error in submitting
+    ExpectedFileFactory.create_batch(
+        5, audit_state=AuditState.NOT_SUBMITTED, study=study
+    )
+    mock_client.bulk_upsert_expected_files.side_effect = Exception
+    with pytest.raises(Exception):
+        task.submit_study_for_audit(study.pk)

tests/storage_analyses/test_expected_file_tasks.py

@@ -0,0 +1,64 @@
+import pytest
+
+from creator.studies.factories import StudyFactory
+from creator.storage_analyses.factories import ExpectedFileFactory
+from creator.storage_analyses import utils
+from creator.storage_analyses.models import ExpectedFile
+
+from ..files.fixtures import make_file_upload_manifest
+
+
+def test_chunked_dataframe_reader(db, mocker, make_file_upload_manifest):
+    """
+    Test utils.chunked_dataframe_reader
+    """
+    # Success
+    version = make_file_upload_manifest(nrows=100)
+    for i, df in enumerate(
+        utils.chunked_dataframe_reader(version, batch_size=10)
+    ):
+        assert df.shape[0] == 10
+    assert i == 9
+
+    # Unknown file format
+    version = make_file_upload_manifest(filename="foo.bar")
+    with pytest.raises(IOError):
+        next(utils.chunked_dataframe_reader(version))
+
+    # Problem reading data
+    mock_logger = mocker.patch("creator.storage_analyses.utils.logger")
+    with pytest.raises(Exception):
+        next(utils.chunked_dataframe_reader(version, batchsize="foo"))
+        assert mock_logger.exception.call_count == 1
+
+
+def test_batched_queryset_iterator(db):
+    """
+    Test utils.batched_queryset_iterator
+    """
+    files = ExpectedFileFactory.create_batch(10)
+    for i, batch in (
+        enumerate(utils.batched_queryset_iterator(ExpectedFile.objects, 5))
+    ):
+        assert len(batch) == 5
+    assert i == 1
+
+
+def test_bulk_upsert_expected_files(db):
+    """
+    Test utils.bulk_upsert_expected_files
+    """
+    study = StudyFactory()
+    files = [
+        {
+            "study": study,
+            "source_file_name": f"myfile{i}.tsv",
+            "hash": f"foobar{i}",
+            "hash_algorithm": "MD5",
+            "size": i * 100
+
+        } for i in range(5)
+    ]
+    assert ExpectedFile.objects.count() == 0
+    utils.bulk_upsert_expected_files(files)
+    assert ExpectedFile.objects.count() == 5