deploy: 7c1ea72

modelscope · Oct 16, 2023 · 4f5cd62 · 4f5cd62
1 parent 1fef0de
commit 4f5cd62
Show file tree

Hide file tree

Showing 104 changed files with 1,724 additions and 1,381 deletions.
diff --git a/.buildinfo b/.buildinfo
@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: ba986cc40075a47c0b766fcda6a61ae4
+config: d12e13dcdf1e8a0872e01cb8a7af3cff
 tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/.doctrees/data_juicer.analysis.doctree b/.doctrees/data_juicer.analysis.doctree
diff --git a/.doctrees/data_juicer.config.doctree b/.doctrees/data_juicer.config.doctree
diff --git a/.doctrees/data_juicer.core.doctree b/.doctrees/data_juicer.core.doctree
diff --git a/.doctrees/data_juicer.doctree b/.doctrees/data_juicer.doctree
diff --git a/.doctrees/data_juicer.format.doctree b/.doctrees/data_juicer.format.doctree
diff --git a/.doctrees/data_juicer.ops.common.doctree b/.doctrees/data_juicer.ops.common.doctree
diff --git a/.doctrees/data_juicer.ops.deduplicator.doctree b/.doctrees/data_juicer.ops.deduplicator.doctree
diff --git a/.doctrees/data_juicer.ops.doctree b/.doctrees/data_juicer.ops.doctree
diff --git a/.doctrees/data_juicer.ops.filter.doctree b/.doctrees/data_juicer.ops.filter.doctree
diff --git a/.doctrees/data_juicer.ops.mapper.doctree b/.doctrees/data_juicer.ops.mapper.doctree
diff --git a/.doctrees/data_juicer.ops.selector.doctree b/.doctrees/data_juicer.ops.selector.doctree
diff --git a/.doctrees/data_juicer.utils.doctree b/.doctrees/data_juicer.utils.doctree
diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle
diff --git a/.doctrees/index.doctree b/.doctrees/index.doctree
diff --git a/.doctrees/modules.doctree b/.doctrees/modules.doctree
diff --git a/LICENSE b/LICENSE
diff --git a/README.md b/README.md
diff --git a/_modules/data_juicer/analysis/column_wise_analysis.html b/_modules/data_juicer/analysis/column_wise_analysis.html
@@ -3,16 +3,16 @@
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>data_juicer.analysis.column_wise_analysis &mdash; data_juicer 0.1.0 documentation</title>
+  <title>data_juicer.analysis.column_wise_analysis &mdash; data_juicer 0.1.2 documentation</title>
       <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
   <!--[if lt IE 9]>
     <script src="../../../_static/js/html5shiv.min.js"></script>
   <![endif]-->
 
-        <script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
-        <script src="../../../_static/doctools.js"></script>
-        <script src="../../../_static/sphinx_highlight.js"></script>
+        <script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js?v=b0099a1c"></script>
+        <script src="../../../_static/doctools.js?v=888ff710"></script>
+        <script src="../../../_static/sphinx_highlight.js?v=4825356b"></script>
     <script src="../../../_static/js/theme.js"></script>
     <link rel="index" title="Index" href="../../../genindex.html" />
     <link rel="search" title="Search" href="../../../search.html" /> 
@@ -30,7 +30,7 @@
             data_juicer
           </a>
               <div class="version">
-                0.1.0
+                0.1.2
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
@@ -75,6 +75,8 @@ <h1>Source code for data_juicer.analysis.column_wise_analysis</h1><div class="hi
 <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
 <span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 
+<span class="kn">from</span> <span class="nn">data_juicer.utils.constant</span> <span class="kn">import</span> <span class="n">Fields</span>
+
 <span class="kn">from</span> <span class="nn">.overall_analysis</span> <span class="kn">import</span> <span class="n">OverallAnalysis</span>
 
 
@@ -135,7 +137,7 @@ <h1>Source code for data_juicer.analysis.column_wise_analysis</h1><div class="hi
 <span class="sd">        :param save_stats_in_one_file: whether save all analysis figures of all</span>
 <span class="sd">            stats into one image file</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">stats</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;stats&#39;</span><span class="p">])</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">stats</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">dataset</span><span class="p">[</span><span class="n">Fields</span><span class="o">.</span><span class="n">stats</span><span class="p">])</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">output_path</span> <span class="o">=</span> <span class="n">output_path</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">output_path</span><span class="p">):</span>
             <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">output_path</span><span class="p">)</span>