Upload tokenizer

tokenizer.py

@@ -79,6 +79,7 @@ class BinnedOmicTokenizer(PreTrainedTokenizer):
             gene_expr = gene_expr / self.normalization_factor
 
         token_ids = np.digitize(gene_expr, self.bin_edges).astype(int)
+        token_ids = np.clip(token_ids, 0, self.n_expressions_bins - 1)
         token_ids[gene_expr == 0.0] = 0
 
         if self.prepend_cls_token:

tokenizer_config.json

@@ -1,17 +1,16 @@
 {
-  "tokenizer_class": "BinnedOmicTokenizer",
-  "n_expressions_bins": 64,
-  "min_omic_value": 0.0,
-  "max_omic_value": 1.0,
-  "use_max_normalization": true,
-  "normalization_factor": 5.547176906585117,
-  "prepend_cls_token": false,
-  "fixed_sequence_length": null,
-  "unpadded_length": null,
+  "added_tokens_decoder": {},
   "auto_map": {
     "AutoTokenizer": [
       "tokenizer.BinnedOmicTokenizer",
       null
     ]
-  }
-}
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<cls>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "BinnedOmicTokenizer"
+}