# R dplyr
library(tidyverse)
library(tictoc)
# Import the two datasets using readr's read_csv funciton
tic()
table_1 = read_csv(file = 'data/table_1.csv')## Rows: 148916 Columns: 7
## ── Column specification ───────────────────────────────────────────────────────
## Delimiter: ","
## chr (7): pmid, doi, title, source, language, pubtype, pubmodel
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 148916 Columns: 6
## ── Column specification ───────────────────────────────────────────────────────
## Delimiter: ","
## chr (5): pmid, authors, year_pub, has_data, oa
## dbl (1): cited
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 13751 Columns: 4
## ── Column specification ───────────────────────────────────────────────────────
## Delimiter: ","
## chr (2): city, country
## dbl (2): lng, lat
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## 0.588 sec elapsed
Remove articles with no pmid and no DOI, and all articles before 1975
Merge the two datasets, pmid is unique for each paper
## 0.035 sec elapsed
## # A tibble: 108,721 × 12
## pmid doi title source language pubtype pubmodel authors year_pub has_data
## <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr>
## 1 32837… 10.1… The … MED eng "{\"pu… "\"Prin… <AUTHO… 2020 N
## 2 32196… 10.1… Hypo… MED eng "{\"pu… "\"Prin… <AUTHO… 2020 N
## 3 32778… 10.1… The … MED eng "{\"pu… "\"Prin… <AUTHO… 2020 N
## 4 32730… 10.1… A co… MED eng "{\"pu… "\"Prin… <AUTHO… 2020 N
## 5 32394… 10.1… A gl… MED eng "{\"pu… "\"Prin… <AUTHO… 2020 N
## 6 32257… 10.1… COVI… MED eng "{\"pu… "\"Elec… <AUTHO… 2020 N
## 7 32720… 10.1… The … MED eng "{\"pu… "\"Prin… <AUTHO… 2020 N
## 8 32196… 10.1… Labo… MED eng "{\"pu… "\"Prin… <AUTHO… 2020 Y
## 9 32656… 10.1… The … MED eng "{\"pu… "\"Prin… <AUTHO… 2020 N
## 10 32732… 10.1… Cycl… MED eng "{\"pu… "\"Prin… <AUTHO… 2020 N
## # ℹ 108,711 more rows
## # ℹ 2 more variables: oa <chr>, cited <dbl>
plot distribution for the log(number of authors +1)
How many papers contains ‘deep learning’ or ‘machine learning’ and ‘neural network’ (also with a ‘s’ for neural networks) in their title ? Create a binary variable to save this information. What is the mean of authors for ML papers and non-ML papers ? Transform has_data and oa into binary variable also, what is the share of ML paper that are oa ?
# R dplyr
tic()
final_table = final_table %>%
mutate(title = tolower(title),
ML = ifelse(str_detect(title,'deep learning|machine learning|neural networks?'), 1, 0),
has_data = ifelse(has_data == 'Y', 1, 0),
oa = ifelse(oa == 'Y', 1, 0))
toc()## 2.346 sec elapsed
## [1] 222
## [1] 77847
## [1] 23980
## # A tibble: 2 × 2
## ML mean_nb_aut
## <dbl> <dbl>
## 1 0 5.77
## 2 1 6.06
## 0.005 sec elapsed
## # A tibble: 2 × 3
## ML n citation
## <dbl> <dbl> <dbl>
## 1 0 0.727 11.3
## 2 1 0.802 1.05
## 0.005 sec elapsed
Clean up pub_type, for simplicity just get the first type
# R dplyr
tic()
final_table = final_table %>%
mutate(pub_type = str_match_all(pubtype, '\\[\\"(.*?)\\"')[[1]][2])
toc()## 1.211 sec elapsed
## # A tibble: 107,103 × 2
## # Rowwise:
## pubtype pub_type
## <chr> <chr>
## 1 "{\"pubType\": [\"review-article\", \"Review\", \"Journal Article\"… review-…
## 2 "{\"pubType\": [\"review-article\", \"Journal Article\"]}" review-…
## 3 "{\"pubType\": [\"review-article\", \"Review\", \"Journal Article\"… review-…
## 4 "{\"pubType\": [\"review-article\", \"Review\", \"Journal Article\"… review-…
## 5 "{\"pubType\": [\"review-article\", \"Review\", \"Journal Article\"… review-…
## 6 "{\"pubType\": [\"review-article\", \"Review\", \"Journal Article\"… review-…
## 7 "{\"pubType\": [\"review-article\", \"Review\", \"Journal Article\"… review-…
## 8 "{\"pubType\": [\"review-article\", \"Review\", \"Journal Article\"… review-…
## 9 "{\"pubType\": [\"Editorial\"]}" Editori…
## 10 "{\"pubType\": [\"Editorial\"]}" Editori…
## # ℹ 107,093 more rows
What is the pub type with the highest mean/sd of citation ? (use cited and the cleaned pub_type)
# R dplyr
final_table %>%
group_by(pub_type) %>%
summarize(mean = mean(cited),
sd = sd(cited)) %>%
arrange(desc(mean))## # A tibble: 83 × 3
## pub_type mean sd
## <chr> <dbl> <dbl>
## 1 Consensus Development Conference 94.1 392.
## 2 Research Support, American Recovery and Reinvestment Act 64.4 77.2
## 3 Research Support, U.S. Gov't, P.H.S. 54.0 97.1
## 4 Research Support, N.I.H., Intramural 48.5 144.
## 5 Clinical Trial, Phase II 35.3 74.0
## 6 Clinical Conference 30.4 84.3
## 7 Comparative Study 29.3 57.1
## 8 Controlled Clinical Trial 27 49.3
## 9 Research Support, Non-U.S. Gov't 26.3 67.8
## 10 Clinical Trial 24.9 56.6
## # ℹ 73 more rows
final_table %>%
group_by(pub_type) %>%
summarize(mean = mean(cited),
sd = sd(cited)) %>%
arrange(desc(sd))## # A tibble: 83 × 3
## pub_type mean sd
## <chr> <dbl> <dbl>
## 1 Consensus Development Conference 94.1 392.
## 2 Research Support, N.I.H., Intramural 48.5 144.
## 3 Research Support, U.S. Gov't, P.H.S. 54.0 97.1
## 4 editorial 7.28 90.8
## 5 Clinical Conference 30.4 84.3
## 6 Research Support, American Recovery and Reinvestment Act 64.4 77.2
## 7 Clinical Trial, Phase II 35.3 74.0
## 8 Research Support, Non-U.S. Gov't 26.3 67.8
## 9 Video-Audio Media 12 61.6
## 10 Comparative Study 29.3 57.1
## # ℹ 73 more rows
What are the most representative country by year ? In another tibble, keep only pmid and authors, and get the country for each author from the loc_cities.csv. You may want to separate rows for each authors to get all countries involved in the paper. If an author have multiple affiliations, take the first one.
## [1] "<AUTHOR> Das SK Subir Kumar Das<AFFILIATION>Department of Biochemistry, College of Medicine and JNM Hospital, WBUHS, Kalyani, Nadia, West Bengal 741235 India."
countries = tolower(unique(cities$country))
tic()
countries_tibble = final_table %>%
select(pmid,authors) %>%
separate_rows(authors, sep = '<AUTHOR>')
toc()## 2.987 sec elapsed
## # A tibble: 725,396 × 2
## pmid authors
## <chr> <chr>
## 1 32837036 ""
## 2 32837036 " Das SK Subir Kumar Das<AFFILIATION>Department of Biochemistry, Co…
## 3 32196410 ""
## 4 32196410 " Lin L Ling Lin<AFFILIATION>Department of Infectious Diseases, Pek…
## 5 32196410 " Lu L Lianfeng Lu<AFFILIATION>Department of Infectious Diseases, P…
## 6 32196410 " Cao W Wei Cao<AFFILIATION>Department of Infectious Diseases, Peki…
## 7 32196410 " Li T Taisheng Li<AFFILIATION>Department of Infectious Diseases, P…
## 8 32778421 ""
## 9 32778421 " Bchetnia M Mbarka Bchetnia<AFFILIATION>Université du Québec à Chi…
## 10 32778421 " Girard C Catherine Girard<AFFILIATION>Université du Québec à Chic…
## # ℹ 725,386 more rows
tic()
countries_tibble = countries_tibble %>%
rowwise() %>%
filter(authors !="" & !str_detect(authors, '<AFFILIATION>None'))
toc()## 7.707 sec elapsed
## # A tibble: 476,357 × 2
## # Rowwise:
## pmid authors
## <chr> <chr>
## 1 32837036 " Das SK Subir Kumar Das<AFFILIATION>Department of Biochemistry, Co…
## 2 32196410 " Lin L Ling Lin<AFFILIATION>Department of Infectious Diseases, Pek…
## 3 32196410 " Lu L Lianfeng Lu<AFFILIATION>Department of Infectious Diseases, P…
## 4 32196410 " Cao W Wei Cao<AFFILIATION>Department of Infectious Diseases, Peki…
## 5 32196410 " Li T Taisheng Li<AFFILIATION>Department of Infectious Diseases, P…
## 6 32778421 " Bchetnia M Mbarka Bchetnia<AFFILIATION>Université du Québec à Chi…
## 7 32778421 " Girard C Catherine Girard<AFFILIATION>Université du Québec à Chic…
## 8 32778421 " Duchaine C Caroline Duchaine<AFFILIATION>Centre de recherche, Ins…
## 9 32778421 " Laprise C Catherine Laprise<AFFILIATION>Université du Québec à Ch…
## 10 32730205 " Liu J Jie Liu<AFFILIATION>Department of Anesthesiology, Union Hos…
## # ℹ 476,347 more rows
tic()
countries_tibble = countries_tibble %>%
group_by(pmid,authors) %>%
mutate(authors = unlist(str_split(authors, '<AFFILIATION>'))[2]) %>%
filter(!duplicated(pmid, authors))
toc()## 8.662 sec elapsed
## # A tibble: 291,650 × 2
## # Groups: pmid, authors [291,650]
## pmid authors
## <chr> <chr>
## 1 32837036 "Department of Biochemistry, College of Medicine and JNM Hospital, …
## 2 32196410 "Department of Infectious Diseases, Peking Union Medical College Ho…
## 3 32196410 "Department of Infectious Diseases, Peking Union Medical College Ho…
## 4 32778421 "Université du Québec à Chicoutimi (UQAC), Département des sciences…
## 5 32778421 "Centre de recherche, Institut universitaire de cardiologie et de p…
## 6 32778421 "Université du Québec à Chicoutimi (UQAC), Département des sciences…
## 7 32730205 "Department of Anesthesiology, Union Hospital, Tongji Medical Colle…
## 8 32730205 "Department of Anesthesiology, Union Hospital, Tongji Medical Colle…
## 9 32394467 "Department of Immunology, Semnan University of Medical Sciences, S…
## 10 32394467 "Department of Bacteriology and Virology, Semnan University of Medi…
## # ℹ 291,640 more rows
# R dplyr
tic()
countries_tibble = countries_tibble %>%
mutate(authors = str_replace_all(authors, ' USA', 'United States'),
authors = str_replace_all(authors, ' UK', 'United Kingdom'),
authors = str_replace_all(authors, ' Korea', 'South Korea'),
authors = tolower(authors))
toc()## 34.33 sec elapsed
pattern = paste0("\\b(", paste(countries, collapse = "|"), ")\\b")
tic()
countries_tibble <- countries_tibble %>%
group_by(pmid,authors) %>%
mutate(country = str_extract(authors, pattern)) %>%
select(-authors)## Adding missing grouping variables: `authors`
## 29.709 sec elapsed
tic()
countries_tibble = countries_tibble %>%
filter(country != '') %>%
distinct(pmid, country)
toc()## 1.165 sec elapsed
## # A tibble: 249,182 × 3
## # Groups: pmid, authors [249,182]
## authors pmid country
## <chr> <chr> <chr>
## 1 "department of biochemistry, college of medicine and jnm hospi… 3283… india
## 2 "department of infectious diseases, peking union medical colle… 3219… china
## 3 "department of infectious diseases, peking union medical colle… 3219… china
## 4 "université du québec à chicoutimi (uqac), département des sci… 3277… canada
## 5 "centre de recherche, institut universitaire de cardiologie et… 3277… canada
## 6 "université du québec à chicoutimi (uqac), département des sci… 3277… canada
## 7 "department of anesthesiology, union hospital, tongji medical … 3273… china
## 8 "department of anesthesiology, union hospital, tongji medical … 3273… china
## 9 "department of immunology, semnan university of medical scienc… 3239… iran
## 10 "department of bacteriology and virology, semnan university of… 3239… iran
## # ℹ 249,172 more rows
tic()
countries_tibble = left_join(countries_tibble,
final_table[,c('pmid','year_pub')],
by = 'pmid')
toc()## 0.122 sec elapsed
tic()
countries_pub = countries_tibble %>%
group_by(country, year_pub) %>%
summarize(n = length(unique(pmid))) %>%
arrange(desc(n))## `summarise()` has grouped output by 'country'. You can override using the
## `.groups` argument.
## 0.019 sec elapsed
## # A tibble: 1,964 × 3
## # Groups: country [198]
## country year_pub n
## <chr> <chr> <int>
## 1 united states 2020 10670
## 2 china 2020 7894
## 3 united kingdom 2020 6058
## 4 italy 2020 5933
## 5 india 2020 3199
## 6 france 2020 2639
## 7 canada 2020 2342
## 8 germany 2020 2108
## 9 australia 2020 2058
## 10 spain 2020 2014
## # ℹ 1,954 more rows
Select the top 25 of countries involved in coronavirus research since 2001, plot the evolution on a bar chart with plot_ly
Two usefull links:
##
## Attaching package: 'kableExtra'
## The following object is masked from 'package:dplyr':
##
## group_rows
plot_top_n = function(n_top,year){
countries_top = countries_pub %>%
filter(year_pub >= year) %>%
group_by(country) %>%
summarize(n = sum(n)) %>%
arrange(desc(n)) %>%
ungroup() %>%
slice(1:n_top)
table = countries_top %>%
kbl(caption = "Top 25 most active countries in coronavirus research (2001-2021)") %>%
kable_classic(full_width = F, html_font = "Cambria")
graph = countries_pub %>%
filter(country %in% as.matrix(countries_top$country) & year_pub >= year) %>%
plot_ly(x = ~year_pub, y = ~n, color = ~country) %>%
layout(yaxis = list(title = 'Count',type = 'log'), barmode = 'stack')
print(table)
print(graph)
}
plot_top_n(10,2001)## <table class=" lightable-classic" style="font-family: Cambria; width: auto !important; margin-left: auto; margin-right: auto;">
## <caption>Top 25 most active countries in coronavirus research (2001-2021)</caption>
## <thead>
## <tr>
## <th style="text-align:left;"> country </th>
## <th style="text-align:right;"> n </th>
## </tr>
## </thead>
## <tbody>
## <tr>
## <td style="text-align:left;"> united states </td>
## <td style="text-align:right;"> 20582 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> china </td>
## <td style="text-align:right;"> 13242 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> united kingdom </td>
## <td style="text-align:right;"> 8388 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> italy </td>
## <td style="text-align:right;"> 6957 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> france </td>
## <td style="text-align:right;"> 4017 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> india </td>
## <td style="text-align:right;"> 3981 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> canada </td>
## <td style="text-align:right;"> 3922 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> germany </td>
## <td style="text-align:right;"> 3655 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> australia </td>
## <td style="text-align:right;"> 3181 </td>
## </tr>
## <tr>
## <td style="text-align:left;"> spain </td>
## <td style="text-align:right;"> 2852 </td>
## </tr>
## </tbody>
## </table>
## No trace type specified:
## Based on info supplied, a 'bar' trace seems appropriate.
## Read more about this trace type -> https://plotly.com/r/reference/#bar