skimr

<!DOCTYPE html> <html> <head> <meta charset="utf-8" /> <meta name="generator" content="pandoc" /> <meta http-equiv="X-UA-Compatible" content="IE=EDGE" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <title>Column type</title> <script>// Pandoc 2.9 adds attributes on both header and div. We remove the former (to // be compatible with the behavior of Pandoc < 2.8). document.addEventListener('DOMContentLoaded', function(e) { var hs = document.querySelectorAll("div.section[class*='level'] > :first-child"); var i, h, a; for (i = 0; i < hs.length; i++) { h = hs[i]; if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6 a = h.attributes; while (a.length > 0) h.removeAttribute(a[0].name); } }); </script> <style type="text/css"> code{white-space: pre-wrap;} span.smallcaps{font-variant: small-caps;} span.underline{text-decoration: underline;} div.column{display: inline-block; vertical-align: top; width: 50%;} div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} ul.task-list{list-style: none;} </style> <style type="text/css"> code { white-space: pre; } .sourceCode { overflow: visible; } </style> <style type="text/css" data-origin="pandoc"> pre > code.sourceCode { white-space: pre; position: relative; } pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } div.sourceCode { margin: 1em 0; } pre.sourceCode { margin: 0; } @media screen { div.sourceCode { overflow: auto; } } @media print { pre > code.sourceCode { white-space: pre-wrap; } pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } pre.numberSource code > span { position: relative; left: -4em; counter-increment: source-line; } pre.numberSource code > span > a:first-child::before { content: counter(source-line); position: relative; left: -1em; text-align: right; vertical-align: baseline; border: none; display: inline-block; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; padding: 0 4px; width: 4em; color: #aaaaaa; } pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; } div.sourceCode { } @media screen { pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; } } code span.al { color: #ff0000; font-weight: bold; } code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } code span.at { color: #7d9029; } code span.bn { color: #40a070; } code span.bu { color: #008000; } code span.cf { color: #007020; font-weight: bold; } code span.ch { color: #4070a0; } code span.cn { color: #880000; } code span.co { color: #60a0b0; font-style: italic; } code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } code span.do { color: #ba2121; font-style: italic; } code span.dt { color: #902000; } code span.dv { color: #40a070; } code span.er { color: #ff0000; font-weight: bold; } code span.ex { } code span.fl { color: #40a070; } code span.fu { color: #06287e; } code span.im { color: #008000; font-weight: bold; } code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } code span.kw { color: #007020; font-weight: bold; } code span.op { color: #666666; } code span.ot { color: #007020; } code span.pp { color: #bc7a00; } code span.sc { color: #4070a0; } code span.ss { color: #bb6688; } code span.st { color: #4070a0; } code span.va { color: #19177c; } code span.vs { color: #4070a0; } code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } </style> <script> // apply pandoc div.sourceCode style to pre.sourceCode instead (function() { var sheets = document.styleSheets; for (var i = 0; i < sheets.length; i++) { if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue; try { var rules = sheets[i].cssRules; } catch (e) { continue; } var j = 0; while (j < rules.length) { var rule = rules[j]; // check if there is a div.sourceCode rule if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") { j++; continue; } var style = rule.style.cssText; // check if color or background-color is set if (rule.style.color === '' && rule.style.backgroundColor === '') { j++; continue; } // replace div.sourceCode by a pre.sourceCode rule sheets[i].deleteRule(j); sheets[i].insertRule('pre.sourceCode{' + style + '}', j); } } })(); </script> <style type="text/css">body { background-color: #fff; margin: 1em auto; max-width: 700px; overflow: visible; padding-left: 2em; padding-right: 2em; font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; font-size: 14px; line-height: 1.35; } #TOC { clear: both; margin: 0 0 10px 10px; padding: 4px; width: 400px; border: 1px solid #CCCCCC; border-radius: 5px; background-color: #f6f6f6; font-size: 13px; line-height: 1.3; } #TOC .toctitle { font-weight: bold; font-size: 15px; margin-left: 5px; } #TOC ul { padding-left: 40px; margin-left: -1.5em; margin-top: 5px; margin-bottom: 5px; } #TOC ul ul { margin-left: -2em; } #TOC li { line-height: 16px; } table { margin: 1em auto; border-width: 1px; border-color: #DDDDDD; border-style: outset; border-collapse: collapse; } table th { border-width: 2px; padding: 5px; border-style: inset; } table td { border-width: 1px; border-style: inset; line-height: 18px; padding: 5px 5px; } table, table th, table td { border-left-style: none; border-right-style: none; } table thead, table tr.even { background-color: #f7f7f7; } p { margin: 0.5em 0; } blockquote { background-color: #f6f6f6; padding: 0.25em 0.75em; } hr { border-style: solid; border: none; border-top: 1px solid #777; margin: 28px 0; } dl { margin-left: 0; } dl dd { margin-bottom: 13px; margin-left: 13px; } dl dt { font-weight: bold; } ul { margin-top: 0; } ul li { list-style: circle outside; } ul ul { margin-bottom: 0; } pre, code { background-color: #f7f7f7; border-radius: 3px; color: #333; white-space: pre-wrap; } pre { border-radius: 3px; margin: 5px 0px 10px 0px; padding: 10px; } pre:not([class]) { background-color: #f7f7f7; } code { font-family: Consolas, Monaco, 'Courier New', monospace; font-size: 85%; } p > code, li > code { padding: 2px 0px; } div.figure { text-align: center; } img { background-color: #FFFFFF; padding: 2px; border: 1px solid #DDDDDD; border-radius: 3px; border: 1px solid #CCCCCC; margin: 0 5px; } h1 { margin-top: 0; font-size: 35px; line-height: 40px; } h2 { border-bottom: 4px solid #f7f7f7; padding-top: 10px; padding-bottom: 2px; font-size: 145%; } h3 { border-bottom: 2px solid #f7f7f7; padding-top: 10px; font-size: 120%; } h4 { border-bottom: 1px solid #f7f7f7; margin-left: 8px; font-size: 105%; } h5, h6 { border-bottom: 1px solid #ccc; font-size: 105%; } a { color: #0033dd; text-decoration: none; } a:hover { color: #6666ff; } a:visited { color: #800080; } a:visited:hover { color: #BB00BB; } a[href^="http:"] { text-decoration: underline; } a[href^="https:"] { text-decoration: underline; } code > span.kw { color: #555; font-weight: bold; } code > span.dt { color: #902000; } code > span.dv { color: #40a070; } code > span.bn { color: #d14; } code > span.fl { color: #d14; } code > span.ch { color: #d14; } code > span.st { color: #d14; } code > span.co { color: #888888; font-style: italic; } code > span.ot { color: #007020; } code > span.al { color: #ff0000; font-weight: bold; } code > span.fu { color: #900; font-weight: bold; } code > span.er { color: #a61717; background-color: #e3d2d2; } </style> </head> <body> <h1 class="title toc-ignore">Column type</h1> This vignette provides an overview of column type specification with readr. Currently it focuses on how automatic guessing works, but over time we expect to cover more topics. <div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>library(readr)</code></pre></div> <div id="automatic-guessing" class="section level2"> <h2>Automatic guessing</h2> If you don’t explicit specify column types with the <code>col_types</code> argument, readr will attempt to guess them using some simple heuristics. By default, it will inspect 1000 values, evenly spaced from the first to the last row. This is a heuristic designed to always be fast (no matter how large your file is) and, in our experience, does a good job in most cases. If needed, you can request that readr use more rows by supplying the <code>guess_max</code> argument. You can even supply <code>guess_max = Inf</code> to use every row to guess the column types. You might wonder why this isn’t the default. That’s because it’s slow: it has to look at every column twice, once to determine the type and once to parse the value. In most cases, you’re best off supplying <code>col_types</code> yourself. <div id="legacy-behavior" class="section level3"> <h3>Legacy behavior</h3> Column type guessing was substantially worse in the first edition of readr (meaning, prior to v2.0.0), because it always looked at the first 1000 rows, and through some application of Murphy’s Law, it appears that many real csv files have lots of empty values at the start, followed by more “excitement” later in the file. Let’s demonstrate the problem with a slightly tricky file: the column <code>x</code> is mostly empty, but has some numeric data at the very end, in row 1001. <div class="sourceCode" id="cb2"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>tricky_dat <- tibble::tibble( <a href="#cb2-2" aria-hidden="true" tabindex="-1"></a> x = rep(c("", "2"), c(1000, 1)), <a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> y = "y" <a href="#cb2-4" aria-hidden="true" tabindex="-1"></a>) <a href="#cb2-5" aria-hidden="true" tabindex="-1"></a>tfile <- tempfile("tricky-column-type-guessing-", fileext = ".csv") <a href="#cb2-6" aria-hidden="true" tabindex="-1"></a>write_csv(tricky_dat, tfile)</code></pre></div> The first edition parser doesn’t guess the right type for <code>x</code> so the <code>2</code> becomes an <code>NA</code>: <div class="sourceCode" id="cb3"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>df <- with_edition(1, read_csv(tfile)) <a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>#> <a href="#cb3-3" aria-hidden="true" tabindex="-1"></a>#> ── Column specification ──────────────────────────────────────────────────────── <a href="#cb3-4" aria-hidden="true" tabindex="-1"></a>#> cols( <a href="#cb3-5" aria-hidden="true" tabindex="-1"></a>#> x = col_logical(), <a href="#cb3-6" aria-hidden="true" tabindex="-1"></a>#> y = col_character() <a href="#cb3-7" aria-hidden="true" tabindex="-1"></a>#> ) <a href="#cb3-8" aria-hidden="true" tabindex="-1"></a>#> Warning: 1 parsing failure. <a href="#cb3-9" aria-hidden="true" tabindex="-1"></a>#> row col expected actual file <a href="#cb3-10" aria-hidden="true" tabindex="-1"></a>#> 1001 x 1/0/T/F/TRUE/FALSE 2 '/tmp/RtmpcP733b/tricky-column-type-guessing-16dac3c2a523e.csv' <a href="#cb3-11" aria-hidden="true" tabindex="-1"></a>tail(df) <a href="#cb3-12" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 6 × 2 <a href="#cb3-13" aria-hidden="true" tabindex="-1"></a>#> x y <a href="#cb3-14" aria-hidden="true" tabindex="-1"></a>#> <lgl> <chr> <a href="#cb3-15" aria-hidden="true" tabindex="-1"></a>#> 1 NA y <a href="#cb3-16" aria-hidden="true" tabindex="-1"></a>#> 2 NA y <a href="#cb3-17" aria-hidden="true" tabindex="-1"></a>#> 3 NA y <a href="#cb3-18" aria-hidden="true" tabindex="-1"></a>#> 4 NA y <a href="#cb3-19" aria-hidden="true" tabindex="-1"></a>#> 5 NA y <a href="#cb3-20" aria-hidden="true" tabindex="-1"></a>#> 6 NA y</code></pre></div> For this specific case, we can fix the problem by marginally increasing <code>guess_max</code>: <div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>df <- with_edition(1, read_csv(tfile, guess_max = 1001)) <a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>#> <a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>#> ── Column specification ──────────────────────────────────────────────────────── <a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>#> cols( <a href="#cb4-5" aria-hidden="true" tabindex="-1"></a>#> x = col_double(), <a href="#cb4-6" aria-hidden="true" tabindex="-1"></a>#> y = col_character() <a href="#cb4-7" aria-hidden="true" tabindex="-1"></a>#> ) <a href="#cb4-8" aria-hidden="true" tabindex="-1"></a>tail(df) <a href="#cb4-9" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 6 × 2 <a href="#cb4-10" aria-hidden="true" tabindex="-1"></a>#> x y <a href="#cb4-11" aria-hidden="true" tabindex="-1"></a>#> <dbl> <chr> <a href="#cb4-12" aria-hidden="true" tabindex="-1"></a>#> 1 NA y <a href="#cb4-13" aria-hidden="true" tabindex="-1"></a>#> 2 NA y <a href="#cb4-14" aria-hidden="true" tabindex="-1"></a>#> 3 NA y <a href="#cb4-15" aria-hidden="true" tabindex="-1"></a>#> 4 NA y <a href="#cb4-16" aria-hidden="true" tabindex="-1"></a>#> 5 NA y <a href="#cb4-17" aria-hidden="true" tabindex="-1"></a>#> 6 2 y</code></pre></div> Unlike the second edition, we don’t recommend using <code>guess_max = Inf</code> with the legacy parser, because the engine pre-allocates a large amount of memory in the face of this uncertainty. This means that reading with <code>guess_max = Inf</code> can be extremely slow and might even crash your R session. Instead specify the <code>col_types</code>: <div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>df <- with_edition(1, read_csv(tfile, col_types = list(x = col_double()))) <a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>tail(df) <a href="#cb5-3" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 6 × 2 <a href="#cb5-4" aria-hidden="true" tabindex="-1"></a>#> x y <a href="#cb5-5" aria-hidden="true" tabindex="-1"></a>#> <dbl> <chr> <a href="#cb5-6" aria-hidden="true" tabindex="-1"></a>#> 1 NA y <a href="#cb5-7" aria-hidden="true" tabindex="-1"></a>#> 2 NA y <a href="#cb5-8" aria-hidden="true" tabindex="-1"></a>#> 3 NA y <a href="#cb5-9" aria-hidden="true" tabindex="-1"></a>#> 4 NA y <a href="#cb5-10" aria-hidden="true" tabindex="-1"></a>#> 5 NA y <a href="#cb5-11" aria-hidden="true" tabindex="-1"></a>#> 6 2 y</code></pre></div> </div> </div>   <script> (function () { var script = document.createElement("script"); script.type = "text/javascript"; script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; document.getElementsByTagName("head")[0].appendChild(script); })(); </script> </body> </html>