Hopefully CI runs now

.github/workflows/main.yml

@@ -31,15 +31,15 @@ jobs:
         task:
           - name: Test
             run: |
-              pytest -v --color=yes tests/
+              pytest -v --color=yes  -m "not nonci" tests/
 
         include:
           - python: "3.11"
             task:
               name: Lint
               run: ruff check .
 
-          # Removing mypy for now, as it isn't handling async things correctly
+          # Removing mypy for now, as it isn't handling async things correctly and crashing
           # - python: "3.11"
           #   task:
           #     name: Type check

pyproject.toml

@@ -158,3 +158,6 @@ python_classes = [
 ]
 log_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
 log_level = "DEBUG"
+markers = [
+    "nonci: mark test as not intended for CI runs"
+]

tests/test_dataloader.py

@@ -1,6 +1,7 @@
 import unittest
 from functools import partial
 
+import pytest
 from torch.utils.data import DataLoader
 from tqdm import tqdm
 from transformers import AutoProcessor
@@ -14,6 +15,7 @@
 from olmocr.train.dataprep import batch_prepare_data_for_qwen2_training
 
 
+@pytest.mark.nonci
 class TestBatchQueryResponseDataset(unittest.TestCase):
     def testLoadS3(self):
         ds = load_jsonl_into_ds("s3://ai2-oe-data/jakep/openai_batch_data_v2/*.jsonl", first_n_files=3)

tests/test_dataprep.py

@@ -7,6 +7,7 @@
 from unittest.mock import patch
 
 import numpy as np
+import pytest
 import requests
 import torch
 from PIL import Image
@@ -27,6 +28,7 @@
 from olmocr.train.utils import make_dataset
 
 
+@pytest.mark.nonci
 class TestDataprep(unittest.TestCase):
     def testFullDataloader(self):
         processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct")

tests/test_molmo.py

@@ -1,5 +1,6 @@
 import unittest
 
+import pytest
 import requests
 from PIL import Image
 from transformers import (
@@ -10,6 +11,7 @@
 )
 
 
+@pytest.mark.nonci
 class MolmoProcessorTest(unittest.TestCase):
     def test_molmo_demo(self):
         # load the processor

tests/test_sglang.py

@@ -8,13 +8,13 @@
 import json
 import math
 import os
-import tempfile
 import unittest
 from io import BytesIO
 from pathlib import Path
 from unittest.mock import AsyncMock, patch
 
 import numpy as np
+import pytest
 import torch
 import torch.nn.functional as F
 from httpx import AsyncClient
@@ -36,7 +36,7 @@
 )
 
 
-@unittest.skip("Skip these tests when running CI, they are mostly for experimentation")
+@pytest.mark.nonci
 class TestSglangServer(unittest.IsolatedAsyncioTestCase):
     async def asyncSetUp(self):
         # Mock arguments
@@ -110,6 +110,7 @@ async def asyncTearDown(self):
         #     os.rmdir(self.args.workspace)
 
 
+@pytest.mark.nonci
 class TestHuggingFaceModel(unittest.IsolatedAsyncioTestCase):
     async def asyncSetUp(self):
         # Set up the Hugging Face model and tokenizer
@@ -248,6 +249,7 @@ async def asyncTearDown(self):
         torch.cuda.empty_cache()
 
 
+@pytest.mark.nonci
 class RawSGLangTest(unittest.IsolatedAsyncioTestCase):
     def setUp(self):
         # Set up the Hugging Face model and tokenizer