-
Notifications
You must be signed in to change notification settings - Fork 27
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Deployed a80aaa9 with MkDocs version: 1.5.3
- Loading branch information
Unknown
committed
Jan 16, 2024
0 parents
commit 562d0e0
Showing
79 changed files
with
27,457 additions
and
0 deletions.
There are no files selected for viewing
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,353 @@ | ||
| <!DOCTYPE html> | ||
| <html lang="en"> | ||
| <head> | ||
| <meta charset="utf-8"> | ||
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | ||
|
|
||
|
|
||
|
|
||
| <link rel="shortcut icon" href="/img/favicon.ico"> | ||
| <title>Spark Advanced Topics</title> | ||
| <link href="/css/bootstrap.min.css" rel="stylesheet"> | ||
| <link href="/css/font-awesome.min.css" rel="stylesheet"> | ||
| <link href="/css/base.css" rel="stylesheet"> | ||
| <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/github.min.css"> | ||
| <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script> | ||
| <script>hljs.highlightAll();</script> | ||
| </head> | ||
|
|
||
| <body> | ||
| <div class="navbar fixed-top navbar-expand-lg navbar-dark bg-primary"> | ||
| <div class="container"> | ||
| <a class="navbar-brand" href="/.">Spark Advanced Topics</a> | ||
| <!-- Expander button --> | ||
| <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#navbar-collapse"> | ||
| <span class="navbar-toggler-icon"></span> | ||
| </button> | ||
|
|
||
| <!-- Expanded navigation --> | ||
| <div id="navbar-collapse" class="navbar-collapse collapse"> | ||
| <!-- Main navigation --> | ||
| <ul class="nav navbar-nav"> | ||
| <li class="navitem"> | ||
| <a href="/." class="nav-link">Spark Advanced Topics Working Group Documentation</a> | ||
| </li> | ||
| <li class="dropdown"> | ||
| <a href="#" class="nav-link dropdown-toggle" data-toggle="dropdown">Details <b class="caret"></b></a> | ||
| <ul class="dropdown-menu"> | ||
|
|
||
| <li> | ||
| <a href="/details/best-pratice-collect/" class="dropdown-item">Bringing too much data back to the driver (collect and friends)</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/big-broadcast-join/" class="dropdown-item">Too big broadcast joins</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/broadcast-with-disable/" class="dropdown-item">Tables getting broadcasted even when broadcast is disabled</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/class-or-method-not-found/" class="dropdown-item">Class or method not found</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/container-oom/" class="dropdown-item">Container OOMs</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/correlated-column-not-allowed/" class="dropdown-item">spark.sql.AnalysisException: Correlated column is not allowed in predicate</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/driver-max-result-size/" class="dropdown-item">Result size larger than spark.driver.maxResultSize error OR Kryo serialization failed: Buffer overflow.</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-driver-max-result-size/" class="dropdown-item">Result size larger than spark.driver.maxResultsSize error</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-driver-out-of-memory/" class="dropdown-item">Driver ran out of memory</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-driver-stack-overflow/" class="dropdown-item">Driver ran out of memory</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-executor-out-of-disk/" class="dropdown-item">Executor out of disk error</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-executor-out-of-memory/" class="dropdown-item">Executor ran out of memory</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-invalid-file/" class="dropdown-item">Missing Files / File Not Found / Reading past RLE/BitPacking stream</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-job/" class="dropdown-item">Error</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-memory/" class="dropdown-item">Memory Errors</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-other/" class="dropdown-item">Other errors</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-shuffle/" class="dropdown-item">Fetch Failed exceptions</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/error-sql-analysis/" class="dropdown-item">spark.sql.AnalysisException</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/even_partitioning_still_slow/" class="dropdown-item">Even Partitioning Yet Still Slow</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/failed-to-read-non-parquet-file/" class="dropdown-item">Failed to read non-parquet file</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/failure-executor-large-record/" class="dropdown-item">Large record problems can show up in a few different ways.</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/forced-computations/" class="dropdown-item">Force computations</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/key-skew/" class="dropdown-item">Key/Partition Skew</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/notenoughexecs/" class="dropdown-item">Notenoughexecs</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/partial_aggregates/" class="dropdown-item">Partial v.s. Full Aggregates</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/pyudfoom/" class="dropdown-item">PySpark UDF / UDAF OOM</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/read-partition-issue/" class="dropdown-item">Partition at read time</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/revise-bad_partitioning/" class="dropdown-item">Bad Partitioning</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/revise-even_partitioning_still_slow/" class="dropdown-item">Even Partitioning Yet Still Slow</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-executor/" class="dropdown-item">Slow executor</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-job-slow-cluster/" class="dropdown-item">Slow job slow cluster</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-job/" class="dropdown-item">Slow job</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-map/" class="dropdown-item">Slow Map</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-partition_filter_pushdown/" class="dropdown-item">Partition Filters</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-reduce/" class="dropdown-item">Slow reduce</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-regex-tips/" class="dropdown-item">Regular Expression Tips</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-skewed-join/" class="dropdown-item">Skewed Joins</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-skewed-write/" class="dropdown-item">Skewed/Slow Write</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-stage/" class="dropdown-item">Identify the slow stage</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-writes-s3/" class="dropdown-item">Slow writes on S3</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-writes-too-many-files/" class="dropdown-item">Slow writes due to Too many small files</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/slow-writes/" class="dropdown-item">Slow Writes</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/toobigdag/" class="dropdown-item">Too Big DAG (or when iterative algorithms go bump in the night)</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/toofew_tasks/" class="dropdown-item">Toofew tasks</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/toomany_tasks/" class="dropdown-item">Toomany tasks</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/udfslow/" class="dropdown-item">Avoid UDFs for the most part</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/details/write-fails/" class="dropdown-item">Write Fails</a> | ||
| </li> | ||
| </ul> | ||
| </li> | ||
| <li class="dropdown"> | ||
| <a href="#" class="nav-link dropdown-toggle" data-toggle="dropdown">Flowchart <b class="caret"></b></a> | ||
| <ul class="dropdown-menu"> | ||
|
|
||
| <li> | ||
| <a href="/flowchart/" class="dropdown-item">Index</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/flowchart/error/" class="dropdown-item">Error</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/flowchart/shared/" class="dropdown-item">Shared</a> | ||
| </li> | ||
|
|
||
| <li> | ||
| <a href="/flowchart/slow/" class="dropdown-item">Slow</a> | ||
| </li> | ||
| </ul> | ||
| </li> | ||
| </ul> | ||
|
|
||
| <ul class="nav navbar-nav ml-auto"> | ||
| <li class="nav-item"> | ||
| <a href="#" class="nav-link" data-toggle="modal" data-target="#mkdocs_search_modal"> | ||
| <i class="fa fa-search"></i> Search | ||
| </a> | ||
| </li> | ||
| </ul> | ||
| </div> | ||
| </div> | ||
| </div> | ||
|
|
||
| <div class="container"> | ||
| <div class="row"> | ||
|
|
||
| <div class="row-fluid"> | ||
| <div id="main-content" class="span12"> | ||
| <h1 id="404-page-not-found" style="text-align: center">404</h1> | ||
| <p style="text-align: center"><strong>Page not found</strong></p> | ||
| </div> | ||
| </div> | ||
|
|
||
|
|
||
| </div> | ||
| </div> | ||
|
|
||
| <footer class="col-md-12"> | ||
| <hr> | ||
| <p>Copyright various authors 2022-2024, CC-BY-SA 4.0</p> | ||
| <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p> | ||
| </footer> | ||
| <script src="/js/jquery-3.6.0.min.js"></script> | ||
| <script src="/js/bootstrap.min.js"></script> | ||
| <script> | ||
| var base_url = "/", | ||
| shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83}; | ||
| </script> | ||
| <script src="/js/base.js"></script> | ||
| <script src="/search/main.js"></script> | ||
|
|
||
| <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true"> | ||
| <div class="modal-dialog modal-lg"> | ||
| <div class="modal-content"> | ||
| <div class="modal-header"> | ||
| <h4 class="modal-title" id="searchModalLabel">Search</h4> | ||
| <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">×</span><span class="sr-only">Close</span></button> | ||
| </div> | ||
| <div class="modal-body"> | ||
| <p>From here you can search these documents. Enter your search terms below.</p> | ||
| <form> | ||
| <div class="form-group"> | ||
| <input type="search" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here"> | ||
| </div> | ||
| </form> | ||
| <div id="mkdocs-search-results" data-no-results-text="No results found"></div> | ||
| </div> | ||
| <div class="modal-footer"> | ||
| </div> | ||
| </div> | ||
| </div> | ||
| </div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true"> | ||
| <div class="modal-dialog"> | ||
| <div class="modal-content"> | ||
| <div class="modal-header"> | ||
| <h4 class="modal-title" id="keyboardModalLabel">Keyboard Shortcuts</h4> | ||
| <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">×</span><span class="sr-only">Close</span></button> | ||
| </div> | ||
| <div class="modal-body"> | ||
| <table class="table"> | ||
| <thead> | ||
| <tr> | ||
| <th style="width: 20%;">Keys</th> | ||
| <th>Action</th> | ||
| </tr> | ||
| </thead> | ||
| <tbody> | ||
| <tr> | ||
| <td class="help shortcut"><kbd>?</kbd></td> | ||
| <td>Open this help</td> | ||
| </tr> | ||
| <tr> | ||
| <td class="next shortcut"><kbd>n</kbd></td> | ||
| <td>Next page</td> | ||
| </tr> | ||
| <tr> | ||
| <td class="prev shortcut"><kbd>p</kbd></td> | ||
| <td>Previous page</td> | ||
| </tr> | ||
| <tr> | ||
| <td class="search shortcut"><kbd>s</kbd></td> | ||
| <td>Search</td> | ||
| </tr> | ||
| </tbody> | ||
| </table> | ||
| </div> | ||
| <div class="modal-footer"> | ||
| </div> | ||
| </div> | ||
| </div> | ||
| </div> | ||
|
|
||
| </body> | ||
| </html> |
Oops, something went wrong.