skimr

<!DOCTYPE html> <html> <head> <meta charset="utf-8" /> <meta name="generator" content="pandoc" /> <meta http-equiv="X-UA-Compatible" content="IE=EDGE" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <title>Tidy data</title> <script>// Pandoc 2.9 adds attributes on both header and div. We remove the former (to // be compatible with the behavior of Pandoc < 2.8). document.addEventListener('DOMContentLoaded', function(e) { var hs = document.querySelectorAll("div.section[class*='level'] > :first-child"); var i, h, a; for (i = 0; i < hs.length; i++) { h = hs[i]; if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6 a = h.attributes; while (a.length > 0) h.removeAttribute(a[0].name); } }); </script> <style type="text/css"> code{white-space: pre-wrap;} span.smallcaps{font-variant: small-caps;} span.underline{text-decoration: underline;} div.column{display: inline-block; vertical-align: top; width: 50%;} div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} ul.task-list{list-style: none;} </style> <style type="text/css"> code { white-space: pre; } .sourceCode { overflow: visible; } </style> <style type="text/css" data-origin="pandoc"> pre > code.sourceCode { white-space: pre; position: relative; } pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } div.sourceCode { margin: 1em 0; } pre.sourceCode { margin: 0; } @media screen { div.sourceCode { overflow: auto; } } @media print { pre > code.sourceCode { white-space: pre-wrap; } pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } pre.numberSource code > span { position: relative; left: -4em; counter-increment: source-line; } pre.numberSource code > span > a:first-child::before { content: counter(source-line); position: relative; left: -1em; text-align: right; vertical-align: baseline; border: none; display: inline-block; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; padding: 0 4px; width: 4em; color: #aaaaaa; } pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; } div.sourceCode { } @media screen { pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; } } code span.al { color: #ff0000; font-weight: bold; } code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } code span.at { color: #7d9029; } code span.bn { color: #40a070; } code span.bu { color: #008000; } code span.cf { color: #007020; font-weight: bold; } code span.ch { color: #4070a0; } code span.cn { color: #880000; } code span.co { color: #60a0b0; font-style: italic; } code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } code span.do { color: #ba2121; font-style: italic; } code span.dt { color: #902000; } code span.dv { color: #40a070; } code span.er { color: #ff0000; font-weight: bold; } code span.ex { } code span.fl { color: #40a070; } code span.fu { color: #06287e; } code span.im { color: #008000; font-weight: bold; } code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } code span.kw { color: #007020; font-weight: bold; } code span.op { color: #666666; } code span.ot { color: #007020; } code span.pp { color: #bc7a00; } code span.sc { color: #4070a0; } code span.ss { color: #bb6688; } code span.st { color: #4070a0; } code span.va { color: #19177c; } code span.vs { color: #4070a0; } code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } </style> <script> // apply pandoc div.sourceCode style to pre.sourceCode instead (function() { var sheets = document.styleSheets; for (var i = 0; i < sheets.length; i++) { if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue; try { var rules = sheets[i].cssRules; } catch (e) { continue; } var j = 0; while (j < rules.length) { var rule = rules[j]; // check if there is a div.sourceCode rule if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") { j++; continue; } var style = rule.style.cssText; // check if color or background-color is set if (rule.style.color === '' && rule.style.backgroundColor === '') { j++; continue; } // replace div.sourceCode by a pre.sourceCode rule sheets[i].deleteRule(j); sheets[i].insertRule('pre.sourceCode{' + style + '}', j); } } })(); </script> <style type="text/css">body { background-color: #fff; margin: 1em auto; max-width: 700px; overflow: visible; padding-left: 2em; padding-right: 2em; font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; font-size: 14px; line-height: 1.35; } #TOC { clear: both; margin: 0 0 10px 10px; padding: 4px; width: 400px; border: 1px solid #CCCCCC; border-radius: 5px; background-color: #f6f6f6; font-size: 13px; line-height: 1.3; } #TOC .toctitle { font-weight: bold; font-size: 15px; margin-left: 5px; } #TOC ul { padding-left: 40px; margin-left: -1.5em; margin-top: 5px; margin-bottom: 5px; } #TOC ul ul { margin-left: -2em; } #TOC li { line-height: 16px; } table { margin: 1em auto; border-width: 1px; border-color: #DDDDDD; border-style: outset; border-collapse: collapse; } table th { border-width: 2px; padding: 5px; border-style: inset; } table td { border-width: 1px; border-style: inset; line-height: 18px; padding: 5px 5px; } table, table th, table td { border-left-style: none; border-right-style: none; } table thead, table tr.even { background-color: #f7f7f7; } p { margin: 0.5em 0; } blockquote { background-color: #f6f6f6; padding: 0.25em 0.75em; } hr { border-style: solid; border: none; border-top: 1px solid #777; margin: 28px 0; } dl { margin-left: 0; } dl dd { margin-bottom: 13px; margin-left: 13px; } dl dt { font-weight: bold; } ul { margin-top: 0; } ul li { list-style: circle outside; } ul ul { margin-bottom: 0; } pre, code { background-color: #f7f7f7; border-radius: 3px; color: #333; white-space: pre-wrap; } pre { border-radius: 3px; margin: 5px 0px 10px 0px; padding: 10px; } pre:not([class]) { background-color: #f7f7f7; } code { font-family: Consolas, Monaco, 'Courier New', monospace; font-size: 85%; } p > code, li > code { padding: 2px 0px; } div.figure { text-align: center; } img { background-color: #FFFFFF; padding: 2px; border: 1px solid #DDDDDD; border-radius: 3px; border: 1px solid #CCCCCC; margin: 0 5px; } h1 { margin-top: 0; font-size: 35px; line-height: 40px; } h2 { border-bottom: 4px solid #f7f7f7; padding-top: 10px; padding-bottom: 2px; font-size: 145%; } h3 { border-bottom: 2px solid #f7f7f7; padding-top: 10px; font-size: 120%; } h4 { border-bottom: 1px solid #f7f7f7; margin-left: 8px; font-size: 105%; } h5, h6 { border-bottom: 1px solid #ccc; font-size: 105%; } a { color: #0033dd; text-decoration: none; } a:hover { color: #6666ff; } a:visited { color: #800080; } a:visited:hover { color: #BB00BB; } a[href^="http:"] { text-decoration: underline; } a[href^="https:"] { text-decoration: underline; } code > span.kw { color: #555; font-weight: bold; } code > span.dt { color: #902000; } code > span.dv { color: #40a070; } code > span.bn { color: #d14; } code > span.fl { color: #d14; } code > span.ch { color: #d14; } code > span.st { color: #d14; } code > span.co { color: #888888; font-style: italic; } code > span.ot { color: #007020; } code > span.al { color: #ff0000; font-weight: bold; } code > span.fu { color: #900; font-weight: bold; } code > span.er { color: #a61717; background-color: #e3d2d2; } </style> </head> <body> <h1 class="title toc-ignore">Tidy data</h1> (This is an informal and code heavy version of the full <a href="https://vita.had.co.nz/papers/tidy-data.html">tidy data paper</a>. Please refer to that for more details.) <div id="data-tidying" class="section level2"> <h2>Data tidying</h2> It is often said that 80% of data analysis is spent on the cleaning and preparing data. And it’s not just a first step, but it must be repeated many times over the course of analysis as new problems come to light or new data is collected. To get a handle on the problem, this paper focuses on a small, but important, aspect of data cleaning that I call data tidying: structuring datasets to facilitate analysis. The principles of tidy data provide a standard way to organise data values within a dataset. A standard makes initial data cleaning easier because you don’t need to start from scratch and reinvent the wheel every time. The tidy data standard has been designed to facilitate initial exploration and analysis of the data, and to simplify the development of data analysis tools that work well together. Current tools often require translation. You have to spend time munging the output from one tool so you can input it into another. Tidy datasets and tidy tools work hand in hand to make data analysis easier, allowing you to focus on the interesting domain problem, not on the uninteresting logistics of data. </div> <div id="defining" class="section level2"> <h2>Defining tidy data</h2> <blockquote> Happy families are all alike; every unhappy family is unhappy in its own way — Leo Tolstoy </blockquote> Like families, tidy datasets are all alike but every messy dataset is messy in its own way. Tidy datasets provide a standardized way to link the structure of a dataset (its physical layout) with its semantics (its meaning). In this section, I’ll provide some standard vocabulary for describing the structure and semantics of a dataset, and then use those definitions to define tidy data. <div id="data-structure" class="section level3"> <h3>Data structure</h3> Most statistical datasets are data frames made up of rows and columns. The columns are almost always labeled and the rows are sometimes labeled. The following code provides some data about an imaginary classroom in a format commonly seen in the wild. The table has three columns and four rows, and both rows and columns are labeled. <div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>library(tibble) <a href="#cb1-2" aria-hidden="true" tabindex="-1"></a>classroom <- tribble( <a href="#cb1-3" aria-hidden="true" tabindex="-1"></a> ~name, ~quiz1, ~quiz2, ~test1, <a href="#cb1-4" aria-hidden="true" tabindex="-1"></a> "Billy", NA, "D", "C", <a href="#cb1-5" aria-hidden="true" tabindex="-1"></a> "Suzy", "F", NA, NA, <a href="#cb1-6" aria-hidden="true" tabindex="-1"></a> "Lionel", "B", "C", "B", <a href="#cb1-7" aria-hidden="true" tabindex="-1"></a> "Jenny", "A", "A", "B" <a href="#cb1-8" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb1-9" aria-hidden="true" tabindex="-1"></a>classroom <a href="#cb1-10" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 4 × 4 <a href="#cb1-11" aria-hidden="true" tabindex="-1"></a>#> name quiz1 quiz2 test1 <a href="#cb1-12" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <chr> <chr> <a href="#cb1-13" aria-hidden="true" tabindex="-1"></a>#> 1 Billy <NA> D C <a href="#cb1-14" aria-hidden="true" tabindex="-1"></a>#> 2 Suzy F <NA> <NA> <a href="#cb1-15" aria-hidden="true" tabindex="-1"></a>#> 3 Lionel B C B <a href="#cb1-16" aria-hidden="true" tabindex="-1"></a>#> 4 Jenny A A B</code></pre></div> There are many ways to structure the same underlying data. The following table shows the same data as above, but the rows and columns have been transposed. <div class="sourceCode" id="cb2"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>tribble( <a href="#cb2-2" aria-hidden="true" tabindex="-1"></a> ~assessment, ~Billy, ~Suzy, ~Lionel, ~Jenny, <a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> "quiz1", NA, "F", "B", "A", <a href="#cb2-4" aria-hidden="true" tabindex="-1"></a> "quiz2", "D", NA, "C", "A", <a href="#cb2-5" aria-hidden="true" tabindex="-1"></a> "test1", "C", NA, "B", "B" <a href="#cb2-6" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb2-7" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 3 × 5 <a href="#cb2-8" aria-hidden="true" tabindex="-1"></a>#> assessment Billy Suzy Lionel Jenny <a href="#cb2-9" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <chr> <chr> <chr> <a href="#cb2-10" aria-hidden="true" tabindex="-1"></a>#> 1 quiz1 <NA> F B A <a href="#cb2-11" aria-hidden="true" tabindex="-1"></a>#> 2 quiz2 D <NA> C A <a href="#cb2-12" aria-hidden="true" tabindex="-1"></a>#> 3 test1 C <NA> B B</code></pre></div> The data is the same, but the layout is different. Our vocabulary of rows and columns is simply not rich enough to describe why the two tables represent the same data. In addition to appearance, we need a way to describe the underlying semantics, or meaning, of the values displayed in the table. </div> <div id="data-semantics" class="section level3"> <h3>Data semantics</h3> A dataset is a collection of values, usually either numbers (if quantitative) or strings (if qualitative). Values are organised in two ways. Every value belongs to a variable and an observation. A variable contains all values that measure the same underlying attribute (like height, temperature, duration) across units. An observation contains all values measured on the same unit (like a person, or a day, or a race) across attributes. A tidy version of the classroom data looks like this: (you’ll learn how the functions work a little later) <div class="sourceCode" id="cb3"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>library(tidyr) <a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>library(dplyr)</code></pre></div> <div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>classroom2 <- classroom %>% <a href="#cb4-2" aria-hidden="true" tabindex="-1"></a> pivot_longer(quiz1:test1, names_to = "assessment", values_to = "grade") %>% <a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> arrange(name, assessment) <a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>classroom2 <a href="#cb4-5" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 12 × 3 <a href="#cb4-6" aria-hidden="true" tabindex="-1"></a>#> name assessment grade <a href="#cb4-7" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <chr> <a href="#cb4-8" aria-hidden="true" tabindex="-1"></a>#> 1 Billy quiz1 <NA> <a href="#cb4-9" aria-hidden="true" tabindex="-1"></a>#> 2 Billy quiz2 D <a href="#cb4-10" aria-hidden="true" tabindex="-1"></a>#> 3 Billy test1 C <a href="#cb4-11" aria-hidden="true" tabindex="-1"></a>#> 4 Jenny quiz1 A <a href="#cb4-12" aria-hidden="true" tabindex="-1"></a>#> 5 Jenny quiz2 A <a href="#cb4-13" aria-hidden="true" tabindex="-1"></a>#> 6 Jenny test1 B <a href="#cb4-14" aria-hidden="true" tabindex="-1"></a>#> # … with 6 more rows</code></pre></div> This makes the values, variables, and observations more clear. The dataset contains 36 values representing three variables and 12 observations. The variables are: <ol style="list-style-type: decimal"> <li><code>name</code>, with four possible values (Billy, Suzy, Lionel, and Jenny).</li> <li><code>assessment</code>, with three possible values (quiz1, quiz2, and test1).</li> <li><code>grade</code>, with five or six values depending on how you think of the missing value (A, B, C, D, F, NA).</li> </ol> The tidy data frame explicitly tells us the definition of an observation. In this classroom, every combination of <code>name</code> and <code>assessment</code> is a single measured observation. The dataset also informs us of missing values, which can and do have meaning. Billy was absent for the first quiz, but tried to salvage his grade. Suzy failed the first quiz, so she decided to drop the class. To calculate Billy’s final grade, we might replace this missing value with an F (or he might get a second chance to take the quiz). However, if we want to know the class average for Test 1, dropping Suzy’s structural missing value would be more appropriate than imputing a new value. For a given dataset, it’s usually easy to figure out what are observations and what are variables, but it is surprisingly difficult to precisely define variables and observations in general. For example, if the columns in the classroom data were <code>height</code> and <code>weight</code> we would have been happy to call them variables. If the columns were <code>height</code> and <code>width</code>, it would be less clear cut, as we might think of height and width as values of a <code>dimension</code> variable. If the columns were <code>home phone</code> and <code>work phone</code>, we could treat these as two variables, but in a fraud detection environment we might want variables <code>phone number</code> and <code>number type</code> because the use of one phone number for multiple people might suggest fraud. A general rule of thumb is that it is easier to describe functional relationships between variables (e.g., <code>z</code> is a linear combination of <code>x</code> and <code>y</code>, <code>density</code> is the ratio of <code>weight</code> to <code>volume</code>) than between rows, and it is easier to make comparisons between groups of observations (e.g., average of group a vs. average of group b) than between groups of columns. In a given analysis, there may be multiple levels of observation. For example, in a trial of new allergy medication we might have three observational types: demographic data collected from each person (<code>age</code>, <code>sex</code>, <code>race</code>), medical data collected from each person on each day (<code>number of sneezes</code>, <code>redness of eyes</code>), and meteorological data collected on each day (<code>temperature</code>, <code>pollen count</code>). Variables may change over the course of analysis. Often the variables in the raw data are very fine grained, and may add extra modelling complexity for little explanatory gain. For example, many surveys ask variations on the same question to better get at an underlying trait. In early stages of analysis, variables correspond to questions. In later stages, you change focus to traits, computed by averaging together multiple questions. This considerably simplifies analysis because you don’t need a hierarchical model, and you can often pretend that the data is continuous, not discrete. </div> <div id="tidy-data" class="section level3"> <h3>Tidy data</h3> Tidy data is a standard way of mapping the meaning of a dataset to its structure. A dataset is messy or tidy depending on how rows, columns and tables are matched up with observations, variables and types. In tidy data: <ol style="list-style-type: decimal"> <li>Every column is a variable.</li> <li>Every row is an observation.</li> <li>Every cell is a single value.</li> </ol> This is Codd’s 3rd normal form, but with the constraints framed in statistical language, and the focus put on a single dataset rather than the many connected datasets common in relational databases. Messy data is any other arrangement of the data. Tidy data makes it easy for an analyst or a computer to extract needed variables because it provides a standard way of structuring a dataset. Compare the different versions of the classroom data: in the messy version you need to use different strategies to extract different variables. This slows analysis and invites errors. If you consider how many data analysis operations involve all of the values in a variable (every aggregation function), you can see how important it is to extract these values in a simple, standard way. Tidy data is particularly well suited for vectorised programming languages like R, because the layout ensures that values of different variables from the same observation are always paired. While the order of variables and observations does not affect analysis, a good ordering makes it easier to scan the raw values. One way of organising variables is by their role in the analysis: are values fixed by the design of the data collection, or are they measured during the course of the experiment? Fixed variables describe the experimental design and are known in advance. Computer scientists often call fixed variables dimensions, and statisticians usually denote them with subscripts on random variables. Measured variables are what we actually measure in the study. Fixed variables should come first, followed by measured variables, each ordered so that related variables are contiguous. Rows can then be ordered by the first variable, breaking ties with the second and subsequent (fixed) variables. This is the convention adopted by all tabular displays in this paper. </div> </div> <div id="tidying" class="section level2"> <h2>Tidying messy datasets</h2> Real datasets can, and often do, violate the three precepts of tidy data in almost every way imaginable. While occasionally you do get a dataset that you can start analysing immediately, this is the exception, not the rule. This section describes the five most common problems with messy datasets, along with their remedies: <ul> <li>Column headers are values, not variable names.</li> <li>Multiple variables are stored in one column.</li> <li>Variables are stored in both rows and columns.</li> <li>Multiple types of observational units are stored in the same table.</li> <li>A single observational unit is stored in multiple tables.</li> </ul> Surprisingly, most messy datasets, including types of messiness not explicitly described above, can be tidied with a small set of tools: pivoting (longer and wider) and separating. The following sections illustrate each problem with a real dataset that I have encountered, and show how to tidy them. <div id="column-headers-are-values-not-variable-names" class="section level3"> <h3>Column headers are values, not variable names</h3> A common type of messy dataset is tabular data designed for presentation, where variables form both the rows and columns, and column headers are values, not variable names. While I would call this arrangement messy, in some cases it can be extremely useful. It provides efficient storage for completely crossed designs, and it can lead to extremely efficient computation if desired operations can be expressed as matrix operations. The following code shows a subset of a typical dataset of this form. This dataset explores the relationship between income and religion in the US. It comes from a report produced by the Pew Research Center, an American think-tank that collects data on attitudes to topics ranging from religion to the internet, and produces many reports that contain datasets in this format. <div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>relig_income <a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 18 × 11 <a href="#cb5-3" aria-hidden="true" tabindex="-1"></a>#> religion `<$10k` $10-2…¹ $20-3…² $30-4…³ $40-5…⁴ $50-7…⁵ $75-1…⁶ $100-…⁷ <a href="#cb5-4" aria-hidden="true" tabindex="-1"></a>#> <chr> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <a href="#cb5-5" aria-hidden="true" tabindex="-1"></a>#> 1 Agnostic 27 34 60 81 76 137 122 109 <a href="#cb5-6" aria-hidden="true" tabindex="-1"></a>#> 2 Atheist 12 27 37 52 35 70 73 59 <a href="#cb5-7" aria-hidden="true" tabindex="-1"></a>#> 3 Buddhist 27 21 30 34 33 58 62 39 <a href="#cb5-8" aria-hidden="true" tabindex="-1"></a>#> 4 Catholic 418 617 732 670 638 1116 949 792 <a href="#cb5-9" aria-hidden="true" tabindex="-1"></a>#> 5 Don’t know/re… 15 14 15 11 10 35 21 17 <a href="#cb5-10" aria-hidden="true" tabindex="-1"></a>#> 6 Evangelical P… 575 869 1064 982 881 1486 949 723 <a href="#cb5-11" aria-hidden="true" tabindex="-1"></a>#> # … with 12 more rows, 2 more variables: `>150k` <dbl>, <a href="#cb5-12" aria-hidden="true" tabindex="-1"></a>#> # `Don't know/refused` <dbl>, and abbreviated variable names ¹`$10-20k`, <a href="#cb5-13" aria-hidden="true" tabindex="-1"></a>#> # ²`$20-30k`, ³`$30-40k`, ⁴`$40-50k`, ⁵`$50-75k`, ⁶`$75-100k`, ⁷`$100-150k`</code></pre></div> This dataset has three variables, <code>religion</code>, <code>income</code> and <code>frequency</code>. To tidy it, we need to pivot the non-variable columns into a two-column key-value pair. This action is often described as making a wide dataset longer (or taller). When pivoting variables, we need to provide the name of the new key-value columns to create. After defining the columns to pivot (every column except for religion), you will need the name of the key column, which is the name of the variable defined by the values of the column headings. In this case, it’s <code>income</code>. The second argument is the name of the value column, <code>frequency</code>. <div class="sourceCode" id="cb6"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>relig_income %>% <a href="#cb6-2" aria-hidden="true" tabindex="-1"></a> pivot_longer(-religion, names_to = "income", values_to = "frequency") <a href="#cb6-3" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 180 × 3 <a href="#cb6-4" aria-hidden="true" tabindex="-1"></a>#> religion income frequency <a href="#cb6-5" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <dbl> <a href="#cb6-6" aria-hidden="true" tabindex="-1"></a>#> 1 Agnostic <$10k 27 <a href="#cb6-7" aria-hidden="true" tabindex="-1"></a>#> 2 Agnostic $10-20k 34 <a href="#cb6-8" aria-hidden="true" tabindex="-1"></a>#> 3 Agnostic $20-30k 60 <a href="#cb6-9" aria-hidden="true" tabindex="-1"></a>#> 4 Agnostic $30-40k 81 <a href="#cb6-10" aria-hidden="true" tabindex="-1"></a>#> 5 Agnostic $40-50k 76 <a href="#cb6-11" aria-hidden="true" tabindex="-1"></a>#> 6 Agnostic $50-75k 137 <a href="#cb6-12" aria-hidden="true" tabindex="-1"></a>#> # … with 174 more rows</code></pre></div> This form is tidy because each column represents a variable and each row represents an observation, in this case a demographic unit corresponding to a combination of <code>religion</code> and <code>income</code>. This format is also used to record regularly spaced observations over time. For example, the Billboard dataset shown below records the date a song first entered the billboard top 100. It has variables for <code>artist</code>, <code>track</code>, <code>date.entered</code>, <code>rank</code> and <code>week</code>. The rank in each week after it enters the top 100 is recorded in 75 columns, <code>wk1</code> to <code>wk75</code>. This form of storage is not tidy, but it is useful for data entry. It reduces duplication since otherwise each song in each week would need its own row, and song metadata like title and artist would need to be repeated. This will be discussed in more depth in <a href="#multiple-types">multiple types</a>. <div class="sourceCode" id="cb7"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>billboard <a href="#cb7-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 317 × 79 <a href="#cb7-3" aria-hidden="true" tabindex="-1"></a>#> artist track date.ent…¹ wk1 wk2 wk3 wk4 wk5 wk6 wk7 wk8 wk9 <a href="#cb7-4" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <date> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <a href="#cb7-5" aria-hidden="true" tabindex="-1"></a>#> 1 2 Pac Baby… 2000-02-26 87 82 72 77 87 94 99 NA NA <a href="#cb7-6" aria-hidden="true" tabindex="-1"></a>#> 2 2Ge+her The … 2000-09-02 91 87 92 NA NA NA NA NA NA <a href="#cb7-7" aria-hidden="true" tabindex="-1"></a>#> 3 3 Door… Kryp… 2000-04-08 81 70 68 67 66 57 54 53 51 <a href="#cb7-8" aria-hidden="true" tabindex="-1"></a>#> 4 3 Door… Loser 2000-10-21 76 76 72 69 67 65 55 59 62 <a href="#cb7-9" aria-hidden="true" tabindex="-1"></a>#> 5 504 Bo… Wobb… 2000-04-15 57 34 25 17 17 31 36 49 53 <a href="#cb7-10" aria-hidden="true" tabindex="-1"></a>#> 6 98^0 Give… 2000-08-19 51 39 34 26 26 19 2 2 3 <a href="#cb7-11" aria-hidden="true" tabindex="-1"></a>#> # … with 311 more rows, 67 more variables: wk10 <dbl>, wk11 <dbl>, wk12 <dbl>, <a href="#cb7-12" aria-hidden="true" tabindex="-1"></a>#> # wk13 <dbl>, wk14 <dbl>, wk15 <dbl>, wk16 <dbl>, wk17 <dbl>, wk18 <dbl>, <a href="#cb7-13" aria-hidden="true" tabindex="-1"></a>#> # wk19 <dbl>, wk20 <dbl>, wk21 <dbl>, wk22 <dbl>, wk23 <dbl>, wk24 <dbl>, <a href="#cb7-14" aria-hidden="true" tabindex="-1"></a>#> # wk25 <dbl>, wk26 <dbl>, wk27 <dbl>, wk28 <dbl>, wk29 <dbl>, wk30 <dbl>, <a href="#cb7-15" aria-hidden="true" tabindex="-1"></a>#> # wk31 <dbl>, wk32 <dbl>, wk33 <dbl>, wk34 <dbl>, wk35 <dbl>, wk36 <dbl>, <a href="#cb7-16" aria-hidden="true" tabindex="-1"></a>#> # wk37 <dbl>, wk38 <dbl>, wk39 <dbl>, wk40 <dbl>, wk41 <dbl>, wk42 <dbl>, <a href="#cb7-17" aria-hidden="true" tabindex="-1"></a>#> # wk43 <dbl>, wk44 <dbl>, wk45 <dbl>, wk46 <dbl>, wk47 <dbl>, wk48 <dbl>, …</code></pre></div> To tidy this dataset, we first use <code>pivot_longer()</code> to make the dataset longer. We transform the columns from <code>wk1</code> to <code>wk76</code>, making a new column for their names, <code>week</code>, and a new value for their values, <code>rank</code>: <div class="sourceCode" id="cb8"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>billboard2 <- billboard %>% <a href="#cb8-2" aria-hidden="true" tabindex="-1"></a> pivot_longer( <a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> wk1:wk76, <a href="#cb8-4" aria-hidden="true" tabindex="-1"></a> names_to = "week", <a href="#cb8-5" aria-hidden="true" tabindex="-1"></a> values_to = "rank", <a href="#cb8-6" aria-hidden="true" tabindex="-1"></a> values_drop_na = TRUE <a href="#cb8-7" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb8-8" aria-hidden="true" tabindex="-1"></a>billboard2 <a href="#cb8-9" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 5,307 × 5 <a href="#cb8-10" aria-hidden="true" tabindex="-1"></a>#> artist track date.entered week rank <a href="#cb8-11" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <date> <chr> <dbl> <a href="#cb8-12" aria-hidden="true" tabindex="-1"></a>#> 1 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk1 87 <a href="#cb8-13" aria-hidden="true" tabindex="-1"></a>#> 2 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk2 82 <a href="#cb8-14" aria-hidden="true" tabindex="-1"></a>#> 3 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk3 72 <a href="#cb8-15" aria-hidden="true" tabindex="-1"></a>#> 4 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk4 77 <a href="#cb8-16" aria-hidden="true" tabindex="-1"></a>#> 5 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk5 87 <a href="#cb8-17" aria-hidden="true" tabindex="-1"></a>#> 6 2 Pac Baby Don't Cry (Keep... 2000-02-26 wk6 94 <a href="#cb8-18" aria-hidden="true" tabindex="-1"></a>#> # … with 5,301 more rows</code></pre></div> Here we use <code>values_drop_na = TRUE</code> to drop any missing values from the rank column. In this data, missing values represent weeks that the song wasn’t in the charts, so can be safely dropped. In this case it’s also nice to do a little cleaning, converting the week variable to a number, and figuring out the date corresponding to each week on the charts: <div class="sourceCode" id="cb9"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a>billboard3 <- billboard2 %>% <a href="#cb9-2" aria-hidden="true" tabindex="-1"></a> mutate( <a href="#cb9-3" aria-hidden="true" tabindex="-1"></a> week = as.integer(gsub("wk", "", week)), <a href="#cb9-4" aria-hidden="true" tabindex="-1"></a> date = as.Date(date.entered) + 7 * (week - 1), <a href="#cb9-5" aria-hidden="true" tabindex="-1"></a> date.entered = NULL <a href="#cb9-6" aria-hidden="true" tabindex="-1"></a> ) <a href="#cb9-7" aria-hidden="true" tabindex="-1"></a>billboard3 <a href="#cb9-8" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 5,307 × 5 <a href="#cb9-9" aria-hidden="true" tabindex="-1"></a>#> artist track week rank date <a href="#cb9-10" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <int> <dbl> <date> <a href="#cb9-11" aria-hidden="true" tabindex="-1"></a>#> 1 2 Pac Baby Don't Cry (Keep... 1 87 2000-02-26 <a href="#cb9-12" aria-hidden="true" tabindex="-1"></a>#> 2 2 Pac Baby Don't Cry (Keep... 2 82 2000-03-04 <a href="#cb9-13" aria-hidden="true" tabindex="-1"></a>#> 3 2 Pac Baby Don't Cry (Keep... 3 72 2000-03-11 <a href="#cb9-14" aria-hidden="true" tabindex="-1"></a>#> 4 2 Pac Baby Don't Cry (Keep... 4 77 2000-03-18 <a href="#cb9-15" aria-hidden="true" tabindex="-1"></a>#> 5 2 Pac Baby Don't Cry (Keep... 5 87 2000-03-25 <a href="#cb9-16" aria-hidden="true" tabindex="-1"></a>#> 6 2 Pac Baby Don't Cry (Keep... 6 94 2000-04-01 <a href="#cb9-17" aria-hidden="true" tabindex="-1"></a>#> # … with 5,301 more rows</code></pre></div> Finally, it’s always a good idea to sort the data. We could do it by artist, track and week: <div class="sourceCode" id="cb10"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a>billboard3 %>% arrange(artist, track, week) <a href="#cb10-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 5,307 × 5 <a href="#cb10-3" aria-hidden="true" tabindex="-1"></a>#> artist track week rank date <a href="#cb10-4" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <int> <dbl> <date> <a href="#cb10-5" aria-hidden="true" tabindex="-1"></a>#> 1 2 Pac Baby Don't Cry (Keep... 1 87 2000-02-26 <a href="#cb10-6" aria-hidden="true" tabindex="-1"></a>#> 2 2 Pac Baby Don't Cry (Keep... 2 82 2000-03-04 <a href="#cb10-7" aria-hidden="true" tabindex="-1"></a>#> 3 2 Pac Baby Don't Cry (Keep... 3 72 2000-03-11 <a href="#cb10-8" aria-hidden="true" tabindex="-1"></a>#> 4 2 Pac Baby Don't Cry (Keep... 4 77 2000-03-18 <a href="#cb10-9" aria-hidden="true" tabindex="-1"></a>#> 5 2 Pac Baby Don't Cry (Keep... 5 87 2000-03-25 <a href="#cb10-10" aria-hidden="true" tabindex="-1"></a>#> 6 2 Pac Baby Don't Cry (Keep... 6 94 2000-04-01 <a href="#cb10-11" aria-hidden="true" tabindex="-1"></a>#> # … with 5,301 more rows</code></pre></div> Or by date and rank: <div class="sourceCode" id="cb11"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a>billboard3 %>% arrange(date, rank) <a href="#cb11-2" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 5,307 × 5 <a href="#cb11-3" aria-hidden="true" tabindex="-1"></a>#> artist track week rank date <a href="#cb11-4" aria-hidden="true" tabindex="-1"></a>#> <chr> <chr> <int> <dbl> <date> <a href="#cb11-5" aria-hidden="true" tabindex="-1"></a>#> 1 Lonestar Amazed 1 81 1999-06-05 <a href="#cb11-6" aria-hidden="true" tabindex="-1"></a>#> 2 Lonestar Amazed 2 54 1999-06-12 <a href="#cb11-7" aria-hidden="true" tabindex="-1"></a>#> 3 Lonestar Amazed 3 44 1999-06-19 <a href="#cb11-8" aria-hidden="true" tabindex="-1"></a>#> 4 Lonestar Amazed 4 39 1999-06-26 <a href="#cb11-9" aria-hidden="true" tabindex="-1"></a>#> 5 Lonestar Amazed 5 38 1999-07-03 <a href="#cb11-10" aria-hidden="true" tabindex="-1"></a>#> 6 Lonestar Amazed 6 33 1999-07-10 <a href="#cb11-11" aria-hidden="true" tabindex="-1"></a>#> # … with 5,301 more rows</code></pre></div> </div> <div id="multiple-variables-stored-in-one-column" class="section level3"> <h3>Multiple variables stored in one column</h3> After pivoting columns, the key column is sometimes a combination of multiple underlying variable names. This happens in the <code>tb</code> (tuberculosis) dataset, shown below. This dataset comes from the World Health Organisation, and records the counts of confirmed tuberculosis cases by <code>country</code>, <code>year</code>, and demographic group. The demographic groups are broken down by <code>sex</code> (m, f) and <code>age</code> (0-14, 15-25, 25-34, 35-44, 45-54, 55-64, unknown). <div class="sourceCode" id="cb12"><pre class="sourceCode r"><code class="sourceCode r"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a>tb <- as_tibble(read.csv("tb.csv", stringsAsFactors = FALSE)) <a href="#cb12-2" aria-hidden="true" tabindex="-1"></a>tb <a href="#cb12-3" aria-hidden="true" tabindex="-1"></a>#> # A tibble: 5,769 × 22 <a href="#cb12-4" aria-hidden="true" tabindex="-1"></a>#> iso2 year m04 m514 m014 m1524 m2534 m3544 m4554 m5564 m65 mu f04 <a href="#cb12-5" aria-hidden="true" tabindex="-1"></a>#> <chr> <int> <int> <int> <int> <int> <int> <int> <int> <int> <int> <int> <int> <a href="#cb12-6" aria-hidden="true" tabindex="-1"></a>#> 1 AD 1989 NA NA NA NA NA NA NA NA NA NA NA <a href="#cb12-7" aria-hidden="true" tabindex="-1"></a>#> 2 AD 1990 NA NA NA NA NA NA NA NA NA NA NA <a href="#cb12-8" aria-hidden="true" tabindex="-1"></a>#> 3 AD 1991 NA NA NA NA NA NA NA NA NA NA NA <a href="#cb12-9" aria-hidden="true" tabindex="-1"></a>#> 4 AD 1992 NA NA NA NA NA NA NA NA NA NA NA <a href="#cb12-10" aria-hidden="true" tabindex="-1"></a>#> 5 AD 1993 NA NA NA NA NA NA NA NA NA NA NA <a href="#cb12-11" aria-hidden="true" tabindex="-1"></a>#> 6 AD 1994 NA NA NA NA NA NA NA NA NA NA NA <a href="#cb12-12" aria-hidden="true" tabindex="-1"></a>#> # … with 5,763 more rows, and 9 more variables: f514 <int>, f014 <int>, <a href="#cb12-13" aria-hidden="true" tabindex="-1"></a>#> # f1524 <int>, f2534 <int>, f3544 <int>, f4554 <int>, f5564 <int>, f65 <int>, <span i