Análise Exploratória de Dados usando R

library("readr")

## Import dataset
flights <- readr::read_csv("./data/flights.csv.zip")
head(flights, n = 5)

library("tidyr")
library("ggplot2")
library("viridis")

dt <- delays %>% 
   dplyr::mutate(delay = round(delay, 2)) %>% 
   tidyr::unite(col = "flight", origin, dest, sep = "-")

g <- ggplot(aes(x = flight, y = delay), data = dt) +
   geom_bar(stat = "identity") +
   labs(x = "Flight", y = "Delay (min)", title = "Average Delays", 
        subtitle = "Flights from EWR")
plot(g)

g <- ggplot(aes(x = reorder(flight, delay), y = delay, 
                fill = reorder(flight, delay)), data = dt) +
   geom_bar(width = 0.7, stat = "identity", color = "white", size = 0.5) +
   coord_flip() +
   geom_text(aes(label = delay), nudge_y = -4, color = "white", size = 4, 
             fontface = "bold") +
   viridis::scale_fill_viridis(direction = -1, discrete = TRUE, guide = "none") +
   labs(x = "Flight", y = "Delay (min)", title = "Average Delays", 
        subtitle = "Flights from EWR")
plot(g)

dt <- flights %>% 
   dplyr::filter(dep_delay > 0) %>% 
   tidyr::unite("flight", origin, dest, sep = "-") %>% 
   dplyr::group_by(flight) %>% 
   dplyr::summarise(num = n()) %>% 
   dplyr::ungroup() %>% 
   dplyr::arrange(desc(num))
head(dt, n = 5)

dt <- flights %>% 
   na.omit() %>% 
   dplyr::filter(dep_delay > 0) %>% 
   tidyr::unite("flight", origin, dest, sep = "-") %>% 
   dplyr::filter(flight == "JFK-LAX") %>% 
   dplyr::mutate(dep_month = lubridate::month(dep_time)) %>% 
   dplyr::group_by(dep_month) %>% 
   dplyr::summarize(delay_mean = median(dep_delay), delay_sd = IQR(dep_delay), 
                    count = n()) %>% 
   dplyr::ungroup()

g <- ggplot(aes(x = dep_month, y = delay_mean), data = dt) +
   geom_ribbon(aes(ymin = delay_mean - delay_sd, ymax = delay_mean + delay_sd), 
               alpha = 0.2) +
   geom_line(size = 1, color = "#482878FF", alpha = 0.6) +
   geom_point(size = 3, color = "#482878FF", alpha = 0.8) +
   labs(x = "Month", y = "Delay (min)", title = "Delay Trends", 
        subtitle = "Flight from JFK to LAX") +
   theme_bw()
plot(g)

dt <- flights %>% 
   na.omit() %>% 
   tidyr::unite("flight", origin, dest, sep = "-") %>% 
   dplyr::filter(flight == "JFK-LAX") %>% 
   dplyr::mutate(dep_weekday = lubridate::wday(dep_time, label = TRUE, 
                                               abbr = TRUE)) %>%
   dplyr::mutate(dep_month = lubridate::month(dep_time, label = TRUE, 
                                              abbr = TRUE)) %>% 
   dplyr::group_by(dep_month, dep_weekday) %>% 
   dplyr::summarise(rate = sum(dep_delay > 0) / n()) %>% 
   dplyr::ungroup()
head(dt, n = 5)

g <- ggplot(dt, aes(x = dep_month, y = dep_weekday, fill = rate)) +
   geom_tile(color = "gray93", size = 1, alpha = 0.8) +
   geom_text(aes(label = round(rate, 2)), color = "white") +
   viridis::scale_fill_viridis(direction = -1, name = "Rate") +
   guides(fill = guide_colorbar(barwidth = 0.5, barheight = 10)) +
   theme_gray(base_family = "Helvetica", base_size = 14) +
   labs(x = "Month", y = "Weekday")
plot(g)

airports <- readr::read_csv("./data/airports.csv.zip")
head(airports, n = 5)

library("ggmap")

dest <- airports %>% 
   dplyr::filter(faa %in% unique(flights$dest))

map <- ggmap::get_map("United States", zoom = 4, maptype = "terrain", 
                      scale = 2, color = "bw")

g <- ggmap::ggmap(map, extent = "device", darken = 0.1) +
   stat_density2d(data = dest, aes(x = lon, y = lat, fill = ..level.., 
                                   alpha = ..level..), 
                  size = 0.01, bins = 10, geom = "polygon") + 
   geom_density2d(data = dest, aes(x = lon, y = lat, color = ..level..), 
                  size = 0.5, bins = 15, alpha = 0.8) + 
   geom_point(aes(lon, lat), data = dest, size = 2, alpha = 0.8, color = "black") +
   scale_color_gradientn(colors = heat.colors(100)[80:5], guide = "none") + 
   scale_fill_gradientn(colors = heat.colors(100)[80:5], guide = "none") + 
   scale_alpha(range = c(0.1, 0.4), guide = "none")
plot(g)

dt <- flights %>% 
   dplyr::filter(origin == "EWR") %>% 
   dplyr::rename(faa = dest) %>% 
   dplyr::left_join(airports, by = "faa") %>% 
   na.omit() %>% 
   dplyr::rename(dest = faa, dest_lat = lat, dest_lon = lon)
dt <- dt %>% 
   dplyr::select(origin, dest, dest_lat, dest_lon, dep_delay) %>% 
   dplyr::rename(faa = origin) %>% 
   dplyr::left_join(airports, by = "faa") %>% 
   na.omit() %>% 
   dplyr::rename(origin = faa, orig_lat = lat, orig_lon = lon) %>% 
   dplyr::select(starts_with("orig"), starts_with("dest"), dep_delay)
dt <- dt %>%
   dplyr::filter(dep_delay > 0) %>% 
   dplyr::group_by(origin, dest) %>% 
   dplyr::summarise_each(funs(median))

map <- ggmap::get_map("United States", zoom = 4, maptype = "terrain", 
                      scale = 2, color = "color")

g <- ggmap::ggmap(map, extent = "device", darken = 0.5) +
   geom_curve(aes(x = orig_lon, y = orig_lat, xend = dest_lon, yend = dest_lat, 
                  color = dep_delay), data = dt, size = 0.5, curvature = 0.5) +
   geom_point(aes(x = dest_lon, y = dest_lat), data = dt, color = "red") +
   geom_text(data = dt, aes(x = dest_lon, y = dest_lat, label = dest), col = "black", 
             size = 3, nudge_x = .5, nudge_y = .5, fontface = "bold") +
   coord_cartesian() +
   viridis::scale_color_viridis(name = "Avg Delay", option = "B", direction = -1) +
   guides(color = guide_colorbar(barwidth = 15, barheight = 0.5)) +
   theme(legend.position = "bottom")
plot(g)

db <- dplyr::src_postgres(
   dbname = Sys.getenv("POSTGRES_DBNAME"),
   port = Sys.getenv("POSTGRES_PORT"),
   host = Sys.getenv("POSTGRES_HOST"),
   user = Sys.getenv("POSTGRES_USER"),
   password = Sys.getenv("POSTGRES_PASS")
)
my.tbl <- dplyr::tbl(db, "tbl_name")
my.tbl <- dplyr::tbl(db, dplyr::sql("SELECT * FROM tbl_name LIMIT 50"))
dt <- my.tbl %>%
   dplyr::filter() %>% 
   dplyr::mutate() %>% 
   dplyr::select() %>% 
   dplyr::collect()

library("sparklyr")
library("dplyr")

sc <- spark_connect(master="local")
flights <- copy_to(sc, flights, "flights")
dt <- flights %>% 
   dplyr::filter() %>% 
   dplyr::group_by() %>% 
   dplyr::summarise() %>% 
   dplyr::collect()