Commit fdc68a8b by Daniel Lee

docker: adds mysqldb with test data

Downloads a large open dataset from NYC Open Data.
Then converts date columns into the mysql format.
parent 6ea99540
## MySQL with Open Data Set from NYC Open Data (https://data.cityofnewyork.us)
FROM mysql:latest
ENV MYSQL_DATABASE="testdata" \
MYSQL_ROOT_PASSWORD="rootpass" \
MYSQL_USER="grafana" \
MYSQL_PASSWORD="password"
# Install requirement (wget)
RUN apt-get update && apt-get install -y wget && apt-get install unzip
# Fetch NYC Data Set
RUN wget https://data.cityofnewyork.us/download/fpz8-jqf4/application%2Fzip -O /tmp/data.zip && \
unzip -j /tmp/data.zip 311_Service_Requests_from_2011.csv -d /var/lib/mysql-files
ADD import_csv.sql /docker-entrypoint-initdb.d/
EXPOSE 3306
mysql_opendata:
build: blocks/mysql_opendata
environment:
MYSQL_ROOT_PASSWORD: rootpass
MYSQL_DATABASE: testdata
MYSQL_USER: grafana
MYSQL_PASSWORD: password
ports:
- "3306:3306"
use testdata;
DROP TABLE IF EXISTS `nyc_open_data`;
CREATE TABLE IF NOT EXISTS `nyc_open_data` (
UniqueKey bigint(255),
`CreatedDate` varchar(255),
`ClosedDate` varchar(255),
Agency varchar(255),
AgencyName varchar(255),
ComplaintType varchar(255),
Descriptor varchar(255),
LocationType varchar(255),
IncidentZip varchar(255),
IncidentAddress varchar(255),
StreetName varchar(255),
CrossStreet1 varchar(255),
CrossStreet2 varchar(255),
IntersectionStreet1 varchar(255),
IntersectionStreet2 varchar(255),
AddressType varchar(255),
City varchar(255),
Landmark varchar(255),
FacilityType varchar(255),
Status varchar(255),
`DueDate` varchar(255),
ResolutionDescription varchar(2048),
`ResolutionActionUpdatedDate` varchar(255),
CommunityBoard varchar(255),
Borough varchar(255),
XCoordinateStatePlane varchar(255),
YCoordinateStatePlane varchar(255),
ParkFacilityName varchar(255),
ParkBorough varchar(255),
SchoolName varchar(255),
SchoolNumber varchar(255),
SchoolRegion varchar(255),
SchoolCode varchar(255),
SchoolPhoneNumber varchar(255),
SchoolAddress varchar(255),
SchoolCity varchar(255),
SchoolState varchar(255),
SchoolZip varchar(255),
SchoolNotFound varchar(255),
SchoolOrCitywideComplaint varchar(255),
VehicleType varchar(255),
TaxiCompanyBorough varchar(255),
TaxiPickUpLocation varchar(255),
BridgeHighwayName varchar(255),
BridgeHighwayDirection varchar(255),
RoadRamp varchar(255),
BridgeHighwaySegment varchar(255),
GarageLotName varchar(255),
FerryDirection varchar(255),
FerryTerminalName varchar(255),
Latitude varchar(255),
Longitude varchar(255),
Location varchar(255)
);
LOAD DATA INFILE '/var/lib/mysql-files/311_Service_Requests_from_2011.csv' INTO TABLE nyc_open_data FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY ',' IGNORE 1 LINES;
update nyc_open_data set CreatedDate = STR_TO_DATE(CreatedDate, '%m/%d/%Y %r') where CreatedDate <> '';
update nyc_open_data set ClosedDate = STR_TO_DATE(ClosedDate, '%m/%d/%Y %r') where ClosedDate <> '';
update nyc_open_data set DueDate = STR_TO_DATE(DueDate, '%m/%d/%Y %r') where DueDate <> '';
update nyc_open_data set ResolutionActionUpdatedDate = STR_TO_DATE(ResolutionActionUpdatedDate, '%m/%d/%Y %r') where ResolutionActionUpdatedDate <> '';
update nyc_open_data set CreatedDate=null where CreatedDate = '';
update nyc_open_data set ClosedDate=null where ClosedDate = '';
update nyc_open_data set DueDate=null where DueDate = '';
update nyc_open_data set ResolutionActionUpdatedDate=null where ResolutionActionUpdatedDate = '';
alter table nyc_open_data modify CreatedDate datetime null;
alter table nyc_open_data modify ClosedDate datetime null;
alter table nyc_open_data modify DueDate datetime null;
alter table nyc_open_data modify ResolutionActionUpdatedDate datetime null;
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment