skimr

<!DOCTYPE html> <html> <head> <meta charset="utf-8" /> <meta name="generator" content="pandoc" /> <meta http-equiv="X-UA-Compatible" content="IE=EDGE" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <title>Pivoting</title> <script>// Pandoc 2.9 adds attributes on both header and div. We remove the former (to // be compatible with the behavior of Pandoc < 2.8). document.addEventListener('DOMContentLoaded', function(e) { var hs = document.querySelectorAll("div.section[class*='level'] > :first-child"); var i, h, a; for (i = 0; i < hs.length; i++) { h = hs[i]; if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6 a = h.attributes; while (a.length > 0) h.removeAttribute(a[0].name); } }); </script> <style type="text/css"> code{white-space: pre-wrap;} span.smallcaps{font-variant: small-caps;} span.underline{text-decoration: underline;} div.column{display: inline-block; vertical-align: top; width: 50%;} div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} ul.task-list{list-style: none;} </style> <style type="text/css"> code { white-space: pre; } .sourceCode { overflow: visible; } </style> <style type="text/css" data-origin="pandoc"> pre > code.sourceCode { white-space: pre; position: relative; } pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } div.sourceCode { margin: 1em 0; } pre.sourceCode { margin: 0; } @media screen { div.sourceCode { overflow: auto; } } @media print { pre > code.sourceCode { white-space: pre-wrap; } pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } pre.numberSource code > span { position: relative; left: -4em; counter-increment: source-line; } pre.numberSource code > span > a:first-child::before { content: counter(source-line); position: relative; left: -1em; text-align: right; vertical-align: baseline; border: none; display: inline-block; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; padding: 0 4px; width: 4em; color: #aaaaaa; } pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; } div.sourceCode { } @media screen { pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; } } code span.al { color: #ff0000; font-weight: bold; } code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } code span.at { color: #7d9029; } code span.bn { color: #40a070; } code span.bu { color: #008000; } code span.cf { color: #007020; font-weight: bold; } code span.ch { color: #4070a0; } code span.cn { color: #880000; } code span.co { color: #60a0b0; font-style: italic; } code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } code span.do { color: #ba2121; font-style: italic; } code span.dt { color: #902000; } code span.dv { color: #40a070; } code span.er { color: #ff0000; font-weight: bold; } code span.ex { } code span.fl { color: #40a070; } code span.fu { color: #06287e; } code span.im { color: #008000; font-weight: bold; } code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } code span.kw { color: #007020; font-weight: bold; } code span.op { color: #666666; } code span.ot { color: #007020; } code span.pp { color: #bc7a00; } code span.sc { color: #4070a0; } code span.ss { color: #bb6688; } code span.st { color: #4070a0; } code span.va { color: #19177c; } code span.vs { color: #4070a0; } code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } </style> <script> // apply pandoc div.sourceCode style to pre.sourceCode instead (function() { var sheets = document.styleSheets; for (var i = 0; i < sheets.length; i++) { if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue; try { var rules = sheets[i].cssRules; } catch (e) { continue; } var j = 0; while (j < rules.length) { var rule = rules[j]; // check if there is a div.sourceCode rule if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") { j++; continue; } var style = rule.style.cssText; // check if color or background-color is set if (rule.style.color === '' && rule.style.backgroundColor === '') { j++; continue; } // replace div.sourceCode by a pre.sourceCode rule sheets[i].deleteRule(j); sheets[i].insertRule('pre.sourceCode{' + style + '}', j); } } })(); </script> <style type="text/css">body { background-color: #fff; margin: 1em auto; max-width: 700px; overflow: visible; padding-left: 2em; padding-right: 2em; font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; font-size: 14px; line-height: 1.35; } #TOC { clear: both; margin: 0 0 10px 10px; padding: 4px; width: 400px; border: 1px solid #CCCCCC; border-radius: 5px; background-color: #f6f6f6; font-size: 13px; line-height: 1.3; } #TOC .toctitle { font-weight: bold; font-size: 15px; margin-left: 5px; } #TOC ul { padding-left: 40px; margin-left: -1.5em; margin-top: 5px; margin-bottom: 5px; } #TOC ul ul { margin-left: -2em; } #TOC li { line-height: 16px; } table { margin: 1em auto; border-width: 1px; border-color: #DDDDDD; border-style: outset; border-collapse: collapse; } table th { border-width: 2px; padding: 5px; border-style: inset; } table td { border-width: 1px; border-style: inset; line-height: 18px; padding: 5px 5px; } table, table th, table td { border-left-style: none; border-right-style: none; } table thead, table tr.even { background-color: #f7f7f7; } p { margin: 0.5em 0; } blockquote { background-color: #f6f6f6; padding: 0.25em 0.75em; } hr { border-style: solid; border: none; border-top: 1px solid #777; margin: 28px 0; } dl { margin-left: 0; } dl dd { margin-bottom: 13px; margin-left: 13px; } dl dt { font-weight: bold; } ul { margin-top: 0; } ul li { list-style: circle outside; } ul ul { margin-bottom: 0; } pre, code { background-color: #f7f7f7; border-radius: 3px; color: #333; white-space: pre-wrap; } pre { border-radius: 3px; margin: 5px 0px 10px 0px; padding: 10px; } pre:not([class]) { background-color: #f7f7f7; } code { font-family: Consolas, Monaco, 'Courier New', monospace; font-size: 85%; } p > code, li > code { padding: 2px 0px; } div.figure { text-align: center; } img { background-color: #FFFFFF; padding: 2px; border: 1px solid #DDDDDD; border-radius: 3px; border: 1px solid #CCCCCC; margin: 0 5px; } h1 { margin-top: 0; font-size: 35px; line-height: 40px; } h2 { border-bottom: 4px solid #f7f7f7; padding-top: 10px; padding-bottom: 2px; font-size: 145%; } h3 { border-bottom: 2px solid #f7f7f7; padding-top: 10px; font-size: 120%; } h4 { border-bottom: 1px solid #f7f7f7; margin-left: 8px; font-size: 105%; } h5, h6 { border-bottom: 1px solid #ccc; font-size: 105%; } a { color: #0033dd; text-decoration: none; } a:hover { color: #6666ff; } a:visited { color: #800080; } a:visited:hover { color: #BB00BB; } a[href^="http:"] { text-decoration: underline; } a[href^="https:"] { text-decoration: underline; } code > span.kw { color: #555; font-weight: bold; } code > span.dt { color: #902000; } code > span.dv { color: #40a070; } code > span.bn { color: #d14; } code > span.fl { color: #d14; } code > span.ch { color: #d14; } code > span.st { color: #d14; } code > span.co { color: #888888; font-style: italic; } code > span.ot { color: #007020; } code > span.al { color: #ff0000; font-weight: bold; } code > span.fu { color: #900; font-weight: bold; } code > span.er { color: #a61717; background-color: #e3d2d2; } </style> </head> <body> <h1 class="title toc-ignore">Pivoting</h1> <div id="introduction" class="section level2"> <h2>Introduction</h2> This vignette describes the use of the new <code>pivot_longer()</code> and <code>pivot_wider()</code> functions. Their goal is to improve the usability of <code>gather()</code> and <code>spread()</code>, and incorporate state-of-the-art features found in other packages. For some time, it’s been obvious that there is something fundamentally wrong with the design of <code>spread()</code> and <code>gather()</code>. Many people don’t find the names intuitive and find it hard to remember which direction corresponds to spreading and which to gathering. It also seems surprisingly hard to remember the arguments to these functions, meaning that many people (including me!) have to consult the documentation every time. There are two important new features inspired by other R packages that have been advancing reshaping in R: <ul> <li><code>pivot_longer()</code> can work with multiple value variables that may have different types, inspired by the enhanced <code>melt()</code> and <code>dcast()</code> functions provided by the <a href="https://github.com/Rdatatable/data.table/wiki">data.table</a> package by Matt Dowle and Arun Srinivasan.</li> <li><code>pivot_longer()</code> and <code>pivot_wider()</code> can take a data frame that specifies precisely how metadata stored in column names becomes data variables (and vice versa), inspired by the <a href="https://winvector.github.io/cdata/">cdata</a> package by John Mount and Nina Zumel.</li> </ul> In this vignette, you’ll learn the key ideas behind <code>pivot_longer()</code> and <code>pivot_wider()</code> as you see them used to solve a variety of data reshaping challenges ranging from simple to complex. To begin we’ll load some needed packages. In real analysis code, I’d imagine you’d do with the <code>library(tidyverse)</code>, but I can’t do that here since this vignette is embedded in a package. <div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>library(tidyr) <a href="#cb1-2" aria-hidden="true" tabindex="-1"></a>library(dplyr) <a href="#cb1-3" aria-hidden="true" tabindex="-1"></a>library(readr)</code></pre></div> </div> <div id="longer" class="section level2"> <h2>Longer</h2> <code>pivot_longer()</code> makes datasets longer by increasing the number of rows and decreasing the number of columns. I don’t believe it makes sense to describe a dataset as being in “long form”. Length is a relative term, and you can only say (e.g.) that dataset A is longer than dataset B. <code>pivot_longer()</code> is commonly needed to tidy wild-caught datasets as they often optimise for ease of data entry or ease of comparison rather than ease of analysis. The following sections show how to use <code>pivot_longer()</code> for a wide range of realistic datasets. <div id="pew" class="section level3"> <h3>String data in column names</h3> The <code>relig_income</code> dataset stores counts based on a survey which (among other things) asked people about their religion and annual income: <div class="sourceCode" id="cb2"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>relig_income <a href="#cb2-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 18 × 11 <a href="#cb2-3" aria-hidden="true" tabindex="-1"></a>#> religion `<$10k` $10-2…¹ $20-3…² $30-4…³ $40-5…⁴ $50-7…⁵ $75-1…⁶ $100-…⁷ <a href="#cb2-4" aria-hidden="true" tabindex="-1"></a>#> <chr> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <a href="#cb2-5" aria-hidden="true" tabindex="-1"></a>#> 1 Agnostic 27 34 60 81 76 137 122 109 <a href="#cb2-6" aria-hidden="true" tabindex="-1"></a>#> 2 Atheist 12 27 37 52 35 70 73 59 <a href="#cb2-7" aria-hidden="true" tabindex="-1"></a>#> 3 Buddhist 27 21 30 34 33 58 62 39 <a href="#cb2-8" aria-hidden="true" tabindex="-1"></a>#> 4 Catholic 418 617 732 670 638 1116 949 792 <a href="#cb2-9" aria-hidden="true" tabindex="-1"></a>#> 5 Don’t know/r… 15 14 15 11 10 35 21 17 <a href="#cb2-10" aria-hidden="true" tabindex="-1"></a>#> 6 Evangelical … 575 869 1064 982 881 1486 949 723 <a href="#cb2-11" aria-hidden="true" tabindex="-1"></a>#> 7 Hindu 1 9 7 9 11 34 47 48 <a href="#cb2-12" aria-hidden="true" tabindex="-1"></a>#> 8 Historically… 228 244 236 238 197 223 131 81 <a href="#cb2-13" aria-hidden="true" tabindex="-1"></a>#> 9 Jehovah's Wi… 20 27 24 24 21 30 15 11 <a href="#cb2-14" aria-hidden="true" tabindex="-1"></a>#> 10 Jewish 19 19 25 25 30 95 69 87 <a href="#cb2-15" aria-hidden="true" tabindex="-1"></a>#> # … with 8 more rows, 2 more variables: `>150k` <dbl>, <a href="#cb2-16" aria-hidden="true" tabindex="-1"></a>#> # `Don't know/refused` <dbl>, and abbreviated variable names ¹`$10-20k`, <a href="#cb2-17" aria-hidden="true" tabindex="-1"></a>#> # ²`$20-30k`, ³`$30-40k`, ⁴`$40-50k`, ⁵`$50-75k`, ⁶`$75-100k`, ⁷`$100-150k`</code></pre></div> This dataset contains three variables: <ul> <li><code>religion</code>, stored in the rows,</li> <li><code>income</code> spread across the column names, and</li> <li><code>count</code> stored in the cell values.</li> </ul> To tidy it we use <code>pivot_longer()</code>: <div class="sourceCode" id="cb3"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>relig_income %>% <a href="#cb3-2" aria-hidden="true" tabindex="-1"></a> pivot_longer( <a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> cols = !religion, <a href="#cb3-4" aria-hidden="true" tabindex="-1"></a> names_to = "income", <a href="#cb3-5" aria-hidden="true" tabindex="-1"></a> values_to = "count" <a href="#cb3-6" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb3-7" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 180 × 3 <a href="#cb3-8" aria-hidden="true" tabindex="-1"></a>#> religion income count <a href="#cb3-9" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <dbl> <a href="#cb3-10" aria-hidden="true" tabindex="-1"></a>#> 1 Agnostic <$10k 27 <a href="#cb3-11" aria-hidden="true" tabindex="-1"></a>#> 2 Agnostic $10-20k 34 <a href="#cb3-12" aria-hidden="true" tabindex="-1"></a>#> 3 Agnostic $20-30k 60 <a href="#cb3-13" aria-hidden="true" tabindex="-1"></a>#> 4 Agnostic $30-40k 81 <a href="#cb3-14" aria-hidden="true" tabindex="-1"></a>#> 5 Agnostic $40-50k 76 <a href="#cb3-15" aria-hidden="true" tabindex="-1"></a>#> 6 Agnostic $50-75k 137 <a href="#cb3-16" aria-hidden="true" tabindex="-1"></a>#> 7 Agnostic $75-100k 122 <a href="#cb3-17" aria-hidden="true" tabindex="-1"></a>#> 8 Agnostic $100-150k 109 <a href="#cb3-18" aria-hidden="true" tabindex="-1"></a>#> 9 Agnostic >150k 84 <a href="#cb3-19" aria-hidden="true" tabindex="-1"></a>#> 10 Agnostic Don't know/refused 96 <a href="#cb3-20" aria-hidden="true" tabindex="-1"></a>#> # … with 170 more rows</code></pre></div> <ul> <li>The first argument is the dataset to reshape, <code>relig_income</code>.</li> <li><code>cols</code> describes which columns need to be reshaped. In this case, it’s every column apart from <code>religion</code>.</li> <li><code>names_to</code> gives the name of the variable that will be created from the data stored in the column names, i.e. <code>income</code>.</li> <li><code>values_to</code> gives the name of the variable that will be created from the data stored in the cell value, i.e. <code>count</code>.</li> </ul> Neither the <code>names_to</code> nor the <code>values_to</code> column exists in <code>relig_income</code>, so we provide them as strings surrounded by quotes. </div> <div id="billboard" class="section level3"> <h3>Numeric data in column names</h3> The <code>billboard</code> dataset records the billboard rank of songs in the year 2000. It has a form similar to the <code>relig_income</code> data, but the data encoded in the column names is really a number, not a string. <div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>billboard <a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 317 × 79 <a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>#> artist track date.ent…¹ wk1 wk2 wk3 wk4 wk5 wk6 wk7 wk8 wk9 <a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <date> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <a href="#cb4-5" aria-hidden="true" tabindex="-1"></a>#> 1 2 Pac Baby… 2000-02-26 87 82 72 77 87 94 99 NA NA <a href="#cb4-6" aria-hidden="true" tabindex="-1"></a>#> 2 2Ge+h… The … 2000-09-02 91 87 92 NA NA NA NA NA NA <a href="#cb4-7" aria-hidden="true" tabindex="-1"></a>#> 3 3 Doo… Kryp… 2000-04-08 81 70 68 67 66 57 54 53 51 <a href="#cb4-8" aria-hidden="true" tabindex="-1"></a>#> 4 3 Doo… Loser 2000-10-21 76 76 72 69 67 65 55 59 62 <a href="#cb4-9" aria-hidden="true" tabindex="-1"></a>#> 5 504 B… Wobb… 2000-04-15 57 34 25 17 17 31 36 49 53 <a href="#cb4-10" aria-hidden="true" tabindex="-1"></a>#> 6 98^0 Give… 2000-08-19 51 39 34 26 26 19 2 2 3 <a href="#cb4-11" aria-hidden="true" tabindex="-1"></a>#> 7 A*Tee… Danc… 2000-07-08 97 97 96 95 100 NA NA NA NA <a href="#cb4-12" aria-hidden="true" tabindex="-1"></a>#> 8 Aaliy… I Do… 2000-01-29 84 62 51 41 38 35 35 38 38 <a href="#cb4-13" aria-hidden="true" tabindex="-1"></a>#> 9 Aaliy… Try … 2000-03-18 59 53 38 28 21 18 16 14 12 <a href="#cb4-14" aria-hidden="true" tabindex="-1"></a>#> 10 Adams… Open… 2000-08-26 76 76 74 69 68 67 61 58 57 <a href="#cb4-15" aria-hidden="true" tabindex="-1"></a>#> # … with 307 more rows, 67 more variables: wk10 <dbl>, wk11 <dbl>, wk12 <dbl>, <a href="#cb4-16" aria-hidden="true" tabindex="-1"></a>#> # wk13 <dbl>, wk14 <dbl>, wk15 <dbl>, wk16 <dbl>, wk17 <dbl>, wk18 <dbl>, <a href="#cb4-17" aria-hidden="true" tabindex="-1"></a>#> # wk19 <dbl>, wk20 <dbl>, wk21 <dbl>, wk22 <dbl>, wk23 <dbl>, wk24 <dbl>, <a href="#cb4-18" aria-hidden="true" tabindex="-1"></a>#> # wk25 <dbl>, wk26 <dbl>, wk27 <dbl>, wk28 <dbl>, wk29 <dbl>, wk30 <dbl>, <a href="#cb4-19" aria-hidden="true" tabindex="-1"></a>#> # wk31 <dbl>, wk32 <dbl>, wk33 <dbl>, wk34 <dbl>, wk35 <dbl>, wk36 <dbl>, <a href="#cb4-20" aria-hidden="true" tabindex="-1"></a>#> # wk37 <dbl>, wk38 <dbl>, wk39 <dbl>, wk40 <dbl>, wk41 <dbl>, wk42 <dbl>, <a href="#cb4-21" aria-hidden="true" tabindex="-1"></a>#> # wk43 <dbl>, wk44 <dbl>, wk45 <dbl>, wk46 <dbl>, wk47 <dbl>, wk48 <dbl>, …</code></pre></div> We can start with the same basic specification as for the <code>relig_income</code> dataset. Here we want the names to become a variable called <code>week</code>, and the values to become a variable called <code>rank</code>. I also use <code>values_drop_na</code> to drop rows that correspond to missing values. Not every song stays in the charts for all 76 weeks, so the structure of the input data force the creation of unnecessary explicit <code>NA</code>s. <div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>billboard %>% <a href="#cb5-2" aria-hidden="true" tabindex="-1"></a> pivot_longer( <a href="#cb5-3" aria-hidden="true" tabindex="-1"></a> cols = starts_with("wk"), <a href="#cb5-4" aria-hidden="true" tabindex="-1"></a> names_to = "week", <a href="#cb5-5" aria-hidden="true" tabindex="-1"></a> values_to = "rank", <a href="#cb5-6" aria-hidden="true" tabindex="-1"></a> values_drop_na = TRUE <a href="#cb5-7" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb5-8" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 5,307 × 5 <a href="#cb5-9" aria-hidden="true" tabindex="-1"></a>#> artist track date.entered week rank <a href="#cb5-10" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <date> <chr> <dbl> <a href="#cb5-11" aria-hidden="true" tabindex="-1"></a>#> 1 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk1 87 <a href="#cb5-12" aria-hidden="true" tabindex="-1"></a>#> 2 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk2 82 <a href="#cb5-13" aria-hidden="true" tabindex="-1"></a>#> 3 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk3 72 <a href="#cb5-14" aria-hidden="true" tabindex="-1"></a>#> 4 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk4 77 <a href="#cb5-15" aria-hidden="true" tabindex="-1"></a>#> 5 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk5 87 <a href="#cb5-16" aria-hidden="true" tabindex="-1"></a>#> 6 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk6 94 <a href="#cb5-17" aria-hidden="true" tabindex="-1"></a>#> 7 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk7 99 <a href="#cb5-18" aria-hidden="true" tabindex="-1"></a>#> 8 2Ge+her The Hardest Part Of ... 2000-09-02 wk1 91 <a href="#cb5-19" aria-hidden="true" tabindex="-1"></a>#> 9 2Ge+her The Hardest Part Of ... 2000-09-02 wk2 87 <a href="#cb5-20" aria-hidden="true" tabindex="-1"></a>#> 10 2Ge+her The Hardest Part Of ... 2000-09-02 wk3 92 <a href="#cb5-21" aria-hidden="true" tabindex="-1"></a>#> # … with 5,297 more rows</code></pre></div> It would be nice to easily determine how long each song stayed in the charts, but to do that, we’ll need to convert the <code>week</code> variable to an integer. We can do that by using two additional arguments: <code>names_prefix</code> strips off the <code>wk</code> prefix, and <code>names_transform</code> converts <code>week</code> into an integer: <div class="sourceCode" id="cb6"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>billboard %>% <a href="#cb6-2" aria-hidden="true" tabindex="-1"></a> pivot_longer( <a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> cols = starts_with("wk"), <a href="#cb6-4" aria-hidden="true" tabindex="-1"></a> names_to = "week", <a href="#cb6-5" aria-hidden="true" tabindex="-1"></a> names_prefix = "wk", <a href="#cb6-6" aria-hidden="true" tabindex="-1"></a> names_transform = as.integer, <a href="#cb6-7" aria-hidden="true" tabindex="-1"></a> values_to = "rank", <a href="#cb6-8" aria-hidden="true" tabindex="-1"></a> values_drop_na = TRUE, <a href="#cb6-9" aria-hidden="true" tabindex="-1"></a> )</code></pre></div> Alternatively, you could do this with a single argument by using <code>readr::parse_number()</code> which automatically strips non-numeric components: <div class="sourceCode" id="cb7"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>billboard %>% <a href="#cb7-2" aria-hidden="true" tabindex="-1"></a> pivot_longer( <a href="#cb7-3" aria-hidden="true" tabindex="-1"></a> cols = starts_with("wk"), <a href="#cb7-4" aria-hidden="true" tabindex="-1"></a> names_to = "week", <a href="#cb7-5" aria-hidden="true" tabindex="-1"></a> names_transform = readr::parse_number, <a href="#cb7-6" aria-hidden="true" tabindex="-1"></a> values_to = "rank", <a href="#cb7-7" aria-hidden="true" tabindex="-1"></a> values_drop_na = TRUE, <a href="#cb7-8" aria-hidden="true" tabindex="-1"></a> )</code></pre></div> </div> <div id="many-variables-in-column-names" class="section level3"> <h3>Many variables in column names</h3> A more challenging situation occurs when you have multiple variables crammed into the column names. For example, take the <code>who</code> dataset: <div class="sourceCode" id="cb8"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>who <a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 7,240 × 60 <a href="#cb8-3" aria-hidden="true" tabindex="-1"></a>#> country iso2 iso3 year new_s…¹ new_s…² new_s…³ new_s…⁴ new_s…⁵ new_s…⁶ <a href="#cb8-4" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <chr> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <a href="#cb8-5" aria-hidden="true" tabindex="-1"></a>#> 1 Afghanistan AF AFG 1980 NA NA NA NA NA NA <a href="#cb8-6" aria-hidden="true" tabindex="-1"></a>#> 2 Afghanistan AF AFG 1981 NA NA NA NA NA NA <a href="#cb8-7" aria-hidden="true" tabindex="-1"></a>#> 3 Afghanistan AF AFG 1982 NA NA NA NA NA NA <a href="#cb8-8" aria-hidden="true" tabindex="-1"></a>#> 4 Afghanistan AF AFG 1983 NA NA NA NA NA NA <a href="#cb8-9" aria-hidden="true" tabindex="-1"></a>#> 5 Afghanistan AF AFG 1984 NA NA NA NA NA NA <a href="#cb8-10" aria-hidden="true" tabindex="-1"></a>#> 6 Afghanistan AF AFG 1985 NA NA NA NA NA NA <a href="#cb8-11" aria-hidden="true" tabindex="-1"></a>#> 7 Afghanistan AF AFG 1986 NA NA NA NA NA NA <a href="#cb8-12" aria-hidden="true" tabindex="-1"></a>#> 8 Afghanistan AF AFG 1987 NA NA NA NA NA NA <a href="#cb8-13" aria-hidden="true" tabindex="-1"></a>#> 9 Afghanistan AF AFG 1988 NA NA NA NA NA NA <a href="#cb8-14" aria-hidden="true" tabindex="-1"></a>#> 10 Afghanistan AF AFG 1989 NA NA NA NA NA NA <a href="#cb8-15" aria-hidden="true" tabindex="-1"></a>#> # … with 7,230 more rows, 50 more variables: new_sp_m65 <dbl>, <a href="#cb8-16" aria-hidden="true" tabindex="-1"></a>#> # new_sp_f014 <dbl>, new_sp_f1524 <dbl>, new_sp_f2534 <dbl>, <a href="#cb8-17" aria-hidden="true" tabindex="-1"></a>#> # new_sp_f3544 <dbl>, new_sp_f4554 <dbl>, new_sp_f5564 <dbl>, <a href="#cb8-18" aria-hidden="true" tabindex="-1"></a>#> # new_sp_f65 <dbl>, new_sn_m014 <dbl>, new_sn_m1524 <dbl>, <a href="#cb8-19" aria-hidden="true" tabindex="-1"></a>#> # new_sn_m2534 <dbl>, new_sn_m3544 <dbl>, new_sn_m4554 <dbl>, <a href="#cb8-20" aria-hidden="true" tabindex="-1"></a>#> # new_sn_m5564 <dbl>, new_sn_m65 <dbl>, new_sn_f014 <dbl>, <a href="#cb8-21" aria-hidden="true" tabindex="-1"></a>#> # new_sn_f1524 <dbl>, new_sn_f2534 <dbl>, new_sn_f3544 <dbl>, …</code></pre></div> <code>country</code>, <code>iso2</code>, <code>iso3</code>, and <code>year</code> are already variables, so they can be left as is. But the columns from <code>new_sp_m014</code> to <code>newrel_f65</code> encode four variables in their names: <ul> <li>The <code>new_</code>/<code>new</code> prefix indicates these are counts of new cases. This dataset only contains new cases, so we’ll ignore it here because it’s constant.</li> <li><code>sp</code>/<code>rel</code>/<code>ep</code> describe how the case was diagnosed.</li> <li><code>m</code>/<code>f</code> gives the gender.</li> <li><code>014</code>/<code>1524</code>/<code>2535</code>/<code>3544</code>/<code>4554</code>/<code>65</code> supplies the age range.</li> </ul> We can break these variables up by specifying multiple column names in <code>names_to</code>, and then either providing <code>names_sep</code> or <code>names_pattern</code>. Here <code>names_pattern</code> is the most natural fit. It has a similar interface to <code>extract</code>: you give it a regular expression containing groups (defined by <code>()</code>) and it puts each group in a column. <div class="sourceCode" id="cb9"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a>who %>% <a href="#cb9-2" aria-hidden="true" tabindex="-1"></a> pivot_longer( <a href="#cb9-3" aria-hidden="true" tabindex="-1"></a> cols = new_sp_m014:newrel_f65, <a href="#cb9-4" aria-hidden="true" tabindex="-1"></a> names_to = c("diagnosis", "gender", "age"), <a href="#cb9-5" aria-hidden="true" tabindex="-1"></a> names_pattern = "new_?(.*)_(.)(.*)", <a href="#cb9-6" aria-hidden="true" tabindex="-1"></a> values_to = "count" <a href="#cb9-7" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb9-8" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 405,440 × 8 <a href="#cb9-9" aria-hidden="true" tabindex="-1"></a>#> country iso2 iso3 year diagnosis gender age count <a href="#cb9-10" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <chr> <dbl> <chr> <chr> <chr> <dbl> <a href="#cb9-11" aria-hidden="true" tabindex="-1"></a>#> 1 Afghanistan AF AFG 1980 sp m 014 NA <a href="#cb9-12" aria-hidden="true" tabindex="-1"></a>#> 2 Afghanistan AF AFG 1980 sp m 1524 NA <a href="#cb9-13" aria-hidden="true" tabindex="-1"></a>#> 3 Afghanistan AF AFG 1980 sp m 2534 NA <a href="#cb9-14" aria-hidden="true" tabindex="-1"></a>#> 4 Afghanistan AF AFG 1980 sp m 3544 NA <a href="#cb9-15" aria-hidden="true" tabindex="-1"></a>#> 5 Afghanistan AF AFG 1980 sp m 4554 NA <a href="#cb9-16" aria-hidden="true" tabindex="-1"></a>#> 6 Afghanistan AF AFG 1980 sp m 5564 NA <a href="#cb9-17" aria-hidden="true" tabindex="-1"></a>#> 7 Afghanistan AF AFG 1980 sp m 65 NA <a href="#cb9-18" aria-hidden="true" tabindex="-1"></a>#> 8 Afghanistan AF AFG 1980 sp f 014 NA <a href="#cb9-19" aria-hidden="true" tabindex="-1"></a>#> 9 Afghanistan AF AFG 1980 sp f 1524 NA <a href="#cb9-20" aria-hidden="true" tabindex="-1"></a>#> 10 Afghanistan AF AFG 1980 sp f 2534 NA <a href="#cb9-21" aria-hidden="true" tabindex="-1"></a>#> # … with 405,430 more rows</code></pre></div> We could go one step further use readr functions to convert the gender and age to factors. I think this is good practice when you have categorical variables with a known set of values. <div class="sourceCode" id="cb10"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a>who %>% <a href="#cb10-2" aria-hidden="true" tabindex="-1"></a> pivot_longer( <a href="#cb10-3" aria-hidden="true" tabindex="-1"></a> cols = new_sp_m014:newrel_f65, <a href="#cb10-4" aria-hidden="true" tabindex="-1"></a> names_to = c("diagnosis", "gender", "age"), <a href="#cb10-5" aria-hidden="true" tabindex="-1"></a> names_pattern = "new_?(.*)_(.)(.*)", <a href="#cb10-6" aria-hidden="true" tabindex="-1"></a> names_transform = list( <a href="#cb10-7" aria-hidden="true" tabindex="-1"></a> gender = ~ readr::parse_factor(.x, levels = c("f", "m")), <a href="#cb10-8" aria-hidden="true" tabindex="-1"></a> age = ~ readr::parse_factor( <a href="#cb10-9" aria-hidden="true" tabindex="-1"></a> .x, <a href="#cb10-10" aria-hidden="true" tabindex="-1"></a> levels = c("014", "1524", "2534", "3544", "4554", "5564", "65"), <a href="#cb10-11" aria-hidden="true" tabindex="-1"></a> ordered = TRUE <a href="#cb10-12" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb10-13" aria-hidden="true" tabindex="-1"></a> ), <a href="#cb10-14" aria-hidden="true" tabindex="-1"></a> values_to = "count", <a href="#cb10-15" aria-hidden="true" tabindex="-1"></a>)</code></pre></div> Doing it this way is a little more efficient than doing a mutate after the fact, <code>pivot_longer()</code> only has to transform one occurence of each name where a <code>mutate()</code> would need to transform many repetitions. </div> <div id="multiple-observations-per-row" class="section level3"> <h3>Multiple observations per row</h3> So far, we have been working with data frames that have one observation per row, but many important pivoting problems involve multiple observations per row. You can usually recognise this case because name of the column that you want to appear in the output is part of the column name in the input. In this section, you’ll learn how to pivot this sort of data. The following example is adapted from the <a href="https://CRAN.R-project.org/package=data.table/vignettes/datatable-reshape.html">data.table vignette</a>, as inspiration for tidyr’s solution to this problem. <div class="sourceCode" id="cb11"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a>household <a href="#cb11-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 5 × 5 <a href="#cb11-3" aria-hidden="true" tabindex="-1"></a>#> family dob_child1 dob_child2 name_child1 name_child2 <a href="#cb11-4" aria-hidden="true" tabindex="-1"></a>#> <int> <date> <date> <chr> <chr> <a href="#cb11-5" aria-hidden="true" tabindex="-1"></a>#> 1 1 1998-11-26 2000-01-29 Susan Jose <a href="#cb11-6" aria-hidden="true" tabindex="-1"></a>#> 2 2 1996-06-22 NA Mark <NA> <a href="#cb11-7" aria-hidden="true" tabindex="-1"></a>#> 3 3 2002-07-11 2004-04-05 Sam Seth <a href="#cb11-8" aria-hidden="true" tabindex="-1"></a>#> 4 4 2004-10-10 2009-08-27 Craig Khai <a href="#cb11-9" aria-hidden="true" tabindex="-1"></a>#> 5 5 2000-12-05 2005-02-28 Parker Gracie</code></pre></div> Note that we have two pieces of information (or values) for each child: their <code>name</code> and their <code>dob</code> (date of birth). These need to go into separate columns in the result. Again we supply multiple variables to <code>names_to</code>, using <code>names_sep</code> to split up each variable name. Note the special name <code>.value</code>: this tells <code>pivot_longer()</code> that that part of the column name specifies the “value” being measured (which will become a variable in the output). <div class="sourceCode" id="cb12"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a>household %>% <a href="#cb12-2" aria-hidden="true" tabindex="-1"></a> pivot_longer( <a href="#cb12-3" aria-hidden="true" tabindex="-1"></a> cols = !family, <a href="#cb12-4" aria-hidden="true" tabindex="-1"></a> names_to = c(".value", "child"), <a href="#cb12-5" aria-hidden="true" tabindex="-1"></a> names_sep = "_", <a href="#cb12-6" aria-hidden="true" tabindex="-1"></a> values_drop_na = TRUE <a href="#cb12-7" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb12-8" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 9 × 4 <a href="#cb12-9" aria-hidden="true" tabindex="-1"></a>#> family child dob name <a href="#cb12-10" aria-hidden="true" tabindex="-1"></a>#> <int> <chr> <date> <chr> <a href="#cb12-11" aria-hidden="true" tabindex="-1"></a>#> 1 1 child1 1998-11-26 Susan <a href="#cb12-12" aria-hidden="true" tabindex="-1"></a>#> 2 1 child2 2000-01-29 Jose <a href="#cb12-13" aria-hidden="true" tabindex="-1"></a>#> 3 2 child1 1996-06-22 Mark <a href="#cb12-14" aria-hidden="true" tabindex="-1"></a>#> 4 3 child1 2002-07-11 Sam <a href="#cb12-15" aria-hidden="true" tabindex="-1"></a>#> 5 3 child2 2004-04-05 Seth <a href="#cb12-16" aria-hidden="true" tabindex="-1"></a>#> 6 4 child1 2004-10-10 Craig <a href="#cb12-17" aria-hidden="true" tabindex="-1"></a>#> 7 4 child2 2009-08-27 Khai <a href="#cb12-18" aria-hidden="true" tabindex="-1"></a>#> 8 5 child1 2000-12-05 Parker <a href="#cb12-19" aria-hidden="true" tabindex="-1"></a>#> 9 5 child2 2005-02-28 Gracie</code></pre></div> Note the use of <code>values_drop_na = TRUE</code>: the input shape forces the creation of explicit missing variables for observations that don’t exist. A similar problem problem also exists in the <code>anscombe</code> dataset built in to base R: <div class="sourceCode" id="cb13"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a>anscombe <a href="#cb13-2" aria-hidden="true" tabindex="-1"></a>#> x1 x2 x3 x4 y1 y2 y3 y4 <a href="#cb13-3" aria-hidden="true" tabindex="-1"></a>#> 1 10 10 10 8 8.04 9.14 7.46 6.58 <a href="#cb13-4" aria-hidden="true" tabindex="-1"></a>#> 2 8 8 8 8 6.95 8.14 6.77 5.76 <a href="#cb13-5" aria-hidden="true" tabindex="-1"></a>#> 3 13 13 13 8 7.58 8.74 12.74 7.71 <a href="#cb13-6" aria-hidden="true" tabindex="-1"></a>#> 4 9 9 9 8 8.81 8.77 7.11 8.84 <a href="#cb13-7" aria-hidden="true" tabindex="-1"></a>#> 5 11 11 11 8 8.33 9.26 7.81 8.47 <a href="#cb13-8" aria-hidden="true" tabindex="-1"></a>#> 6 14 14 14 8 9.96 8.10 8.84 7.04 <a href="#cb13-9" aria-hidden="true" tabindex="-1"></a>#> 7 6 6 6 8 7.24 6.13 6.08 5.25 <a href="#cb13-10" aria-hidden="true" tabindex="-1"></a>#> 8 4 4 4 19 4.26 3.10 5.39 12.50 <a href="#cb13-11" aria-hidden="true" tabindex="-1"></a>#> 9 12 12 12 8 10.84 9.13 8.15 5.56 <a href="#cb13-12" aria-hidden="true" tabindex="-1"></a>#> 10 7 7 7 8 4.82 7.26 6.42 7.91 <a href="#cb13-13" aria