skimr

<!DOCTYPE html> <html> <head> <meta charset="utf-8" /> <meta name="generator" content="pandoc" /> <meta http-equiv="X-UA-Compatible" content="IE=EDGE" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <title>Nested data</title> <script>// Pandoc 2.9 adds attributes on both header and div. We remove the former (to // be compatible with the behavior of Pandoc < 2.8). document.addEventListener('DOMContentLoaded', function(e) { var hs = document.querySelectorAll("div.section[class*='level'] > :first-child"); var i, h, a; for (i = 0; i < hs.length; i++) { h = hs[i]; if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6 a = h.attributes; while (a.length > 0) h.removeAttribute(a[0].name); } }); </script> <style type="text/css"> code{white-space: pre-wrap;} span.smallcaps{font-variant: small-caps;} span.underline{text-decoration: underline;} div.column{display: inline-block; vertical-align: top; width: 50%;} div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} ul.task-list{list-style: none;} </style> <style type="text/css"> code { white-space: pre; } .sourceCode { overflow: visible; } </style> <style type="text/css" data-origin="pandoc"> pre > code.sourceCode { white-space: pre; position: relative; } pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } div.sourceCode { margin: 1em 0; } pre.sourceCode { margin: 0; } @media screen { div.sourceCode { overflow: auto; } } @media print { pre > code.sourceCode { white-space: pre-wrap; } pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } pre.numberSource code > span { position: relative; left: -4em; counter-increment: source-line; } pre.numberSource code > span > a:first-child::before { content: counter(source-line); position: relative; left: -1em; text-align: right; vertical-align: baseline; border: none; display: inline-block; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; padding: 0 4px; width: 4em; color: #aaaaaa; } pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; } div.sourceCode { } @media screen { pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; } } code span.al { color: #ff0000; font-weight: bold; } code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } code span.at { color: #7d9029; } code span.bn { color: #40a070; } code span.bu { color: #008000; } code span.cf { color: #007020; font-weight: bold; } code span.ch { color: #4070a0; } code span.cn { color: #880000; } code span.co { color: #60a0b0; font-style: italic; } code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } code span.do { color: #ba2121; font-style: italic; } code span.dt { color: #902000; } code span.dv { color: #40a070; } code span.er { color: #ff0000; font-weight: bold; } code span.ex { } code span.fl { color: #40a070; } code span.fu { color: #06287e; } code span.im { color: #008000; font-weight: bold; } code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } code span.kw { color: #007020; font-weight: bold; } code span.op { color: #666666; } code span.ot { color: #007020; } code span.pp { color: #bc7a00; } code span.sc { color: #4070a0; } code span.ss { color: #bb6688; } code span.st { color: #4070a0; } code span.va { color: #19177c; } code span.vs { color: #4070a0; } code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } </style> <script> // apply pandoc div.sourceCode style to pre.sourceCode instead (function() { var sheets = document.styleSheets; for (var i = 0; i < sheets.length; i++) { if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue; try { var rules = sheets[i].cssRules; } catch (e) { continue; } var j = 0; while (j < rules.length) { var rule = rules[j]; // check if there is a div.sourceCode rule if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") { j++; continue; } var style = rule.style.cssText; // check if color or background-color is set if (rule.style.color === '' && rule.style.backgroundColor === '') { j++; continue; } // replace div.sourceCode by a pre.sourceCode rule sheets[i].deleteRule(j); sheets[i].insertRule('pre.sourceCode{' + style + '}', j); } } })(); </script> <style type="text/css">body { background-color: #fff; margin: 1em auto; max-width: 700px; overflow: visible; padding-left: 2em; padding-right: 2em; font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; font-size: 14px; line-height: 1.35; } #TOC { clear: both; margin: 0 0 10px 10px; padding: 4px; width: 400px; border: 1px solid #CCCCCC; border-radius: 5px; background-color: #f6f6f6; font-size: 13px; line-height: 1.3; } #TOC .toctitle { font-weight: bold; font-size: 15px; margin-left: 5px; } #TOC ul { padding-left: 40px; margin-left: -1.5em; margin-top: 5px; margin-bottom: 5px; } #TOC ul ul { margin-left: -2em; } #TOC li { line-height: 16px; } table { margin: 1em auto; border-width: 1px; border-color: #DDDDDD; border-style: outset; border-collapse: collapse; } table th { border-width: 2px; padding: 5px; border-style: inset; } table td { border-width: 1px; border-style: inset; line-height: 18px; padding: 5px 5px; } table, table th, table td { border-left-style: none; border-right-style: none; } table thead, table tr.even { background-color: #f7f7f7; } p { margin: 0.5em 0; } blockquote { background-color: #f6f6f6; padding: 0.25em 0.75em; } hr { border-style: solid; border: none; border-top: 1px solid #777; margin: 28px 0; } dl { margin-left: 0; } dl dd { margin-bottom: 13px; margin-left: 13px; } dl dt { font-weight: bold; } ul { margin-top: 0; } ul li { list-style: circle outside; } ul ul { margin-bottom: 0; } pre, code { background-color: #f7f7f7; border-radius: 3px; color: #333; white-space: pre-wrap; } pre { border-radius: 3px; margin: 5px 0px 10px 0px; padding: 10px; } pre:not([class]) { background-color: #f7f7f7; } code { font-family: Consolas, Monaco, 'Courier New', monospace; font-size: 85%; } p > code, li > code { padding: 2px 0px; } div.figure { text-align: center; } img { background-color: #FFFFFF; padding: 2px; border: 1px solid #DDDDDD; border-radius: 3px; border: 1px solid #CCCCCC; margin: 0 5px; } h1 { margin-top: 0; font-size: 35px; line-height: 40px; } h2 { border-bottom: 4px solid #f7f7f7; padding-top: 10px; padding-bottom: 2px; font-size: 145%; } h3 { border-bottom: 2px solid #f7f7f7; padding-top: 10px; font-size: 120%; } h4 { border-bottom: 1px solid #f7f7f7; margin-left: 8px; font-size: 105%; } h5, h6 { border-bottom: 1px solid #ccc; font-size: 105%; } a { color: #0033dd; text-decoration: none; } a:hover { color: #6666ff; } a:visited { color: #800080; } a:visited:hover { color: #BB00BB; } a[href^="http:"] { text-decoration: underline; } a[href^="https:"] { text-decoration: underline; } code > span.kw { color: #555; font-weight: bold; } code > span.dt { color: #902000; } code > span.dv { color: #40a070; } code > span.bn { color: #d14; } code > span.fl { color: #d14; } code > span.ch { color: #d14; } code > span.st { color: #d14; } code > span.co { color: #888888; font-style: italic; } code > span.ot { color: #007020; } code > span.al { color: #ff0000; font-weight: bold; } code > span.fu { color: #900; font-weight: bold; } code > span.er { color: #a61717; background-color: #e3d2d2; } </style> </head> <body> <h1 class="title toc-ignore">Nested data</h1> <div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>library(tidyr) <a href="#cb1-2" aria-hidden="true" tabindex="-1"></a>library(dplyr) <a href="#cb1-3" aria-hidden="true" tabindex="-1"></a>library(purrr)</code></pre></div> <div id="basics" class="section level2"> <h2>Basics</h2> A nested data frame is a data frame where one (or more) columns is a list of data frames. You can create simple nested data frames by hand: <div class="sourceCode" id="cb2"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>df1 <- tibble( <a href="#cb2-2" aria-hidden="true" tabindex="-1"></a> g = c(1, 2, 3), <a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> data = list( <a href="#cb2-4" aria-hidden="true" tabindex="-1"></a> tibble(x = 1, y = 2), <a href="#cb2-5" aria-hidden="true" tabindex="-1"></a> tibble(x = 4:5, y = 6:7), <a href="#cb2-6" aria-hidden="true" tabindex="-1"></a> tibble(x = 10) <a href="#cb2-7" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb2-8" aria-hidden="true" tabindex="-1"></a>) <a href="#cb2-9" aria-hidden="true" tabindex="-1"></a> <a href="#cb2-10" aria-hidden="true" tabindex="-1"></a>df1 <a href="#cb2-11" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 3 × 2 <a href="#cb2-12" aria-hidden="true" tabindex="-1"></a>#> g data <a href="#cb2-13" aria-hidden="true" tabindex="-1"></a>#> <dbl> <list> <a href="#cb2-14" aria-hidden="true" tabindex="-1"></a>#> 1 1 <tibble [1 × 2]> <a href="#cb2-15" aria-hidden="true" tabindex="-1"></a>#> 2 2 <tibble [2 × 2]> <a href="#cb2-16" aria-hidden="true" tabindex="-1"></a>#> 3 3 <tibble [1 × 1]></code></pre></div> (It is possible to create list-columns in regular data frames, not just in tibbles, but it’s considerably more work because the default behaviour of <code>data.frame()</code> is to treat lists as lists of columns.) But more commonly you’ll create them with <code>tidyr::nest()</code>: <div class="sourceCode" id="cb3"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>df2 <- tribble( <a href="#cb3-2" aria-hidden="true" tabindex="-1"></a> ~g, ~x, ~y, <a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> 1, 1, 2, <a href="#cb3-4" aria-hidden="true" tabindex="-1"></a> 2, 4, 6, <a href="#cb3-5" aria-hidden="true" tabindex="-1"></a> 2, 5, 7, <a href="#cb3-6" aria-hidden="true" tabindex="-1"></a> 3, 10, NA <a href="#cb3-7" aria-hidden="true" tabindex="-1"></a>) <a href="#cb3-8" aria-hidden="true" tabindex="-1"></a>df2 %>% nest(data = c(x, y)) <a href="#cb3-9" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 3 × 2 <a href="#cb3-10" aria-hidden="true" tabindex="-1"></a>#> g data <a href="#cb3-11" aria-hidden="true" tabindex="-1"></a>#> <dbl> <list> <a href="#cb3-12" aria-hidden="true" tabindex="-1"></a>#> 1 1 <tibble [1 × 2]> <a href="#cb3-13" aria-hidden="true" tabindex="-1"></a>#> 2 2 <tibble [2 × 2]> <a href="#cb3-14" aria-hidden="true" tabindex="-1"></a>#> 3 3 <tibble [1 × 2]></code></pre></div> <code>nest()</code> specifies which variables should be nested inside; an alternative is to use <code>dplyr::group_by()</code> to describe which variables should be kept outside. <div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>df2 %>% group_by(g) %>% nest() <a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 3 × 2 <a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>#> # Groups: g [3] <a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>#> g data <a href="#cb4-5" aria-hidden="true" tabindex="-1"></a>#> <dbl> <list> <a href="#cb4-6" aria-hidden="true" tabindex="-1"></a>#> 1 1 <tibble [1 × 2]> <a href="#cb4-7" aria-hidden="true" tabindex="-1"></a>#> 2 2 <tibble [2 × 2]> <a href="#cb4-8" aria-hidden="true" tabindex="-1"></a>#> 3 3 <tibble [1 × 2]></code></pre></div> I think nesting is easiest to understand in connection to grouped data: each row in the output corresponds to one group in the input. We’ll see shortly this is particularly convenient when you have other per-group objects. The opposite of <code>nest()</code> is <code>unnest()</code>. You give it the name of a list-column containing data frames, and it row-binds the data frames together, repeating the outer columns the right number of times to line up. <div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>df1 %>% unnest(data) <a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 4 × 3 <a href="#cb5-3" aria-hidden="true" tabindex="-1"></a>#> g x y <a href="#cb5-4" aria-hidden="true" tabindex="-1"></a>#> <dbl> <dbl> <dbl> <a href="#cb5-5" aria-hidden="true" tabindex="-1"></a>#> 1 1 1 2 <a href="#cb5-6" aria-hidden="true" tabindex="-1"></a>#> 2 2 4 6 <a href="#cb5-7" aria-hidden="true" tabindex="-1"></a>#> 3 2 5 7 <a href="#cb5-8" aria-hidden="true" tabindex="-1"></a>#> 4 3 10 NA</code></pre></div> </div> <div id="nested-data-and-models" class="section level2"> <h2>Nested data and models</h2> Nested data is a great fit for problems where you have one of something for each group. A common place this arises is when you’re fitting multiple models. <div class="sourceCode" id="cb6"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>mtcars_nested <- mtcars %>% <a href="#cb6-2" aria-hidden="true" tabindex="-1"></a> group_by(cyl) %>% <a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> nest() <a href="#cb6-4" aria-hidden="true" tabindex="-1"></a> <a href="#cb6-5" aria-hidden="true" tabindex="-1"></a>mtcars_nested <a href="#cb6-6" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 3 × 2 <a href="#cb6-7" aria-hidden="true" tabindex="-1"></a>#> # Groups: cyl [3] <a href="#cb6-8" aria-hidden="true" tabindex="-1"></a>#> cyl data <a href="#cb6-9" aria-hidden="true" tabindex="-1"></a>#> <dbl> <list> <a href="#cb6-10" aria-hidden="true" tabindex="-1"></a>#> 1 6 <tibble [7 × 10]> <a href="#cb6-11" aria-hidden="true" tabindex="-1"></a>#> 2 4 <tibble [11 × 10]> <a href="#cb6-12" aria-hidden="true" tabindex="-1"></a>#> 3 8 <tibble [14 × 10]></code></pre></div> Once you have a list of data frames, it’s very natural to produce a list of models: <div class="sourceCode" id="cb7"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>mtcars_nested <- mtcars_nested %>% <a href="#cb7-2" aria-hidden="true" tabindex="-1"></a> mutate(model = map(data, function(df) lm(mpg ~ wt, data = df))) <a href="#cb7-3" aria-hidden="true" tabindex="-1"></a>mtcars_nested <a href="#cb7-4" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 3 × 3 <a href="#cb7-5" aria-hidden="true" tabindex="-1"></a>#> # Groups: cyl [3] <a href="#cb7-6" aria-hidden="true" tabindex="-1"></a>#> cyl data model <a href="#cb7-7" aria-hidden="true" tabindex="-1"></a>#> <dbl> <list> <list> <a href="#cb7-8" aria-hidden="true" tabindex="-1"></a>#> 1 6 <tibble [7 × 10]> <lm> <a href="#cb7-9" aria-hidden="true" tabindex="-1"></a>#> 2 4 <tibble [11 × 10]> <lm> <a href="#cb7-10" aria-hidden="true" tabindex="-1"></a>#> 3 8 <tibble [14 × 10]> <lm></code></pre></div> And then you could even produce a list of predictions: <div class="sourceCode" id="cb8"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>mtcars_nested <- mtcars_nested %>% <a href="#cb8-2" aria-hidden="true" tabindex="-1"></a> mutate(model = map(model, predict)) <a href="#cb8-3" aria-hidden="true" tabindex="-1"></a>mtcars_nested <a href="#cb8-4" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 3 × 3 <a href="#cb8-5" aria-hidden="true" tabindex="-1"></a>#> # Groups: cyl [3] <a href="#cb8-6" aria-hidden="true" tabindex="-1"></a>#> cyl data model <a href="#cb8-7" aria-hidden="true" tabindex="-1"></a>#> <dbl> <list> <list> <a href="#cb8-8" aria-hidden="true" tabindex="-1"></a>#> 1 6 <tibble [7 × 10]> <dbl [7]> <a href="#cb8-9" aria-hidden="true" tabindex="-1"></a>#> 2 4 <tibble [11 × 10]> <dbl [11]> <a href="#cb8-10" aria-hidden="true" tabindex="-1"></a>#> 3 8 <tibble [14 × 10]> <dbl [14]></code></pre></div> This workflow works particularly well in conjunction with <a href="https://broom.tidymodels.org/">broom</a>, which makes it easy to turn models into tidy data frames which can then be <code>unnest()</code>ed to get back to flat data frames. You can see a bigger example in the <a href="https://broom.tidymodels.org/articles/broom_and_dplyr.html">broom and dplyr vignette</a>. </div>   <script> (function () { var script = document.createElement("script"); script.type = "text/javascript"; script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; document.getElementsByTagName("head")[0].appendChild(script); })(); </script> </body> </html>