stack-code-refine.yaml

# global parameters
project_name: 'Data-Juicer-recipes-the-stack'
dataset_path: '/path/to/your/dataset'  # path to your dataset directory or file
export_path: '/path/to/your/dataset.jsonl'

text_key: 'content'

np: 50  # number of subprocess to process your dataset
open_tracer: true

# process schedule
# a list of several process operators with their arguments
process:
  - clean_email_mapper:
  - clean_links_mapper:
  - fix_unicode_mapper:
  - punctuation_normalization_mapper:
  - whitespace_normalization_mapper:
  - clean_copyright_mapper:

  - alphanumeric_filter:  # 18766
      tokenization: false
      min_ratio: 0.2  # < 3sigma (0.3791)
      max_ratio: 0.9163  # 3sigma
  - alphanumeric_filter:  # 146432
      tokenization: true
      min_ratio: 0.546  # 3sigma
      max_ratio: 3.65  # 3sigma
  - average_line_length_filter:  # for code
      min_len: 10  # > 3sigma (0) -- 48790
      max_len: 150  # < 3sigma (15603) -- 233275
  - character_repetition_filter:
      max_ratio: 0.36  # 3sigma -- 346875
  - maximum_line_length_filter:  # for code
      max_len: 1000  # remove 256670 samples
  - text_length_filter:
      max_len: 96714  # 3sigma -- 190006
  - words_num_filter:
      min_num: 20  # remove 1504958 samples
      max_num: 6640  # 3sigma -- remove 179847 samples
  - word_repetition_filter:
      rep_len: 10
      max_ratio: 0.357  # 3sigma -- 598462

  - document_simhash_deduplicator:
      tokenization: space
      window_size: 6
      lowercase: true
      ignore_pattern: '\p{P}'
      num_blocks: 6
      hamming_distance: 4