1. 1 CloudFront 2014.03.15 JAWS DAYS 2014

2. 22 JAWS DAYS 2014 ! ( ) ! ! AWS Amazon Elastic Transcoder 3. 33 JAWS DAYS 2014 Amazon CloudFront ! Contents Delivery Network 4. 44 JAWS DAYS 2014 Europe Amsterdam,Netherlands(2) Dublin,Ireland Frankfurt,Germany(3) London,England(3) Madrid,Spain Marseille,France Milan,Italia Paris,France(2) Stockholm,Sweden Warsaw,Poland Asia Chennai,India HongKong,China(2) Mumbai,India Manila,Philippines Osaka,Japan Seoul,Korea Singapore(2) Sydney,Australia Taipei,Taiwan Tokyo,Japan(2) SouthAmerica SaoPaulo,Brazil RiodeJaneiro,Brazil NorthAmerica Atlanta,GA Ashburn,VA(3) Dallas,TX(2) Hayward,CA Jacksonville,FL LosAngeles,CA(2) Miami,FL NewYork,NY(3) Newark,NJ PaloAlto,CA SanJose,CA Seattle,WA SouthBend,IN St.Louis,MO 201403 51EdgeLocations 5. 55 JAWS DAYS 2014 6. 66 JAWS DAYS 2014 CloudFront ! HTTPS ! Geo Restriction ! Signed URL 7. 77 JAWS DAYS 2014 CloudFront Signed URL ! CloudFront URL (Canned Policy) (Custom Policy) IP 8. 88 JAWS DAYS 2014 CloudFront Signed URL (Canned Policy) http(s)://{CloudFront Distiburion CNAME}/{} ?Expires={(Unix Time)} &Signature={} &KeyPair-Id={CloudFront Key Pair ID} (Custom Policy) http(s)://{CloudFront Distiburion CNAME}/{} ?Policy={JSON} &Signature={} &KeyPair-Id={CloudFront Key Pair ID} URL 9. 99 JAWS DAYS 2014 CloudFront Signed URL Amazon CloudFront SignedURL Distribution OAI IP Origin Access Identity 10. 1010 JAWS DAYS 2014 Geo Restriction CloudFront Signed URL URL URL Amazon CloudFront URL CloudFront SignedURL Distribution OAI HTTPS 11. 1111 JAWS DAYS 2014 12. 1212 JAWS DAYS 2014 HTTP HTTP Live Streaming (HLS) HTTP Dynamic Streaming (HDS) IIS Smooth Streaming 13. 1313 JAWS DAYS 2014 Android 3.2 Android 4.0 14. 1414 JAWS DAYS 2014 ()2013 Google Developers http://developer.android.com/about/dashboards/index.html 79.7% HTTP Live Streaming (HLS) 15. 1515 JAWS DAYS 2014 HLS Web (.m3u8) 01(.ts) 02(.ts) 03(.ts) 04(.ts) 05(.ts) : HTTP/HTTPS AES Encryption #EXTM3U #EXT-X-VERSION:3 #EXT-X-MEDIA-SEQUENCE:0 #EXT-X-ALLOW-CACHE:YES #EXT-X-TARGETDURATION:12 #EXTINF:11.333333, segment_hls_1m00000.ts #EXTINF:11.250000, segment_hls_1m00001.ts #EXTINF:7.500000, segment_hls_1m00002.ts : #EXT-X-ENDLIST (m3u8) GET 16. 1616 JAWS DAYS 2014 HLS URL Amazon CloudFront Amazon S3 SignedURL Distribution OAI URL 17. 1717 JAWS DAYS 2014 HLS #EXTM3U #EXT-X-VERSION:3 #EXT-X-MEDIA-SEQUENCE:0 #EXT-X-ALLOW-CACHE:YES #EXT-X-TARGETDURATION:12 #EXTINF:11.333333, segment_hls_1m00000.ts #EXTINF:11.250000, segment_hls_1m00001.ts #EXTINF:7.500000, segment_hls_1m00002.ts : #EXT-X-ENDLIST #EXTM3U #EXT-X-VERSION:3 #EXT-X-MEDIA-SEQUENCE:0 #EXT-X-ALLOW-CACHE:YES #EXT-X-TARGETDURATION:12 #EXTINF:11.333333, https://xxx.cloudfront.net/hls/segment_hls_1m00000.ts? Signature=Yana7RByw30iPHZQzFKIyqoAsLHMPPeZ~w-7RPuHeVTY06VDg nW7MbNjQSbGkHn9kWPdlFAWCX7g1q9Mk5kORLXMcJwCOCm165~P6ss9 Bj8rMmYNoIj96u7Nm3xzwbFHfCf5WyafA6aX1PoQ2Vgod98TZVhHGuTdA- IuiMz6Ly8_&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9 kMWJ3amwwb3JteW9veC5jbG91ZGZyb250Lm5ldC9obHMvKiIsIkNvbmRpd GlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTM5NDI0Nj MwM319fV19&Key-Pair-Id=APKAIZ4RI4PUMO3SNKLQ #EXTINF:11.250000, https://xxx.cloudfront.net/hls/segment_hls_1m00001.ts? Signature=Yana7RByw30iPHZQzFKIyqoAsLHMPPeZ~w-7RPuHeVTY06VDg nW7MbNjQSbGkHn9kWPdlFAWCX7g1q9Mk5kORLXMcJwCOCm165~P6ss9 Bj8rMmYNoIj96u7Nm3xzwbFHfCf5WyafA6aX1PoQ2Vgod98TZVhHGuTdA- IuiMz6Ly8_&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9 kMWJ3amwwb3JteW9veC5jbG91ZGZyb250Lm5ldC9obHMvKiIsIkNvbmRpd GlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTM5NDI0Nj MwM319fV19&Key-Pair-Id=APKAIZ4RI4PUMO3SNKLQ #EXTINF:7.500000, : #EXT-X-ENDLIST SignedURL Custom Policy https://xxx.cloudfront.net/hls/* 18. 1818 JAWS DAYS 2014 HLS ! 1. #EXT-X-ALLOW-CACHE:NO 2. S3Metadata Cache-Control: max-age=0, s-maxage= 1. 2. s-maxage CloudFront 19. 1919 JAWS DAYS 2014 HLS /Web HTTP/HTTPS GET LIVE LiveEncoder #EXT-X-ENDLIST (.m3u8) 01(.ts) 02(.ts) : 20. 2020 JAWS DAYS 2014 HLS ! CloudFrontDistribution CloudFrontBehavior URL 21. 2121 JAWS DAYS 2014 (*.m3u8)Behavior Restrict Viewer Access(Use Signed URLs URL 22. HLS URL Amazon CloudFront SignedURL Distribution URL LIVE LiveEncoder URL 23. 2323 JAWS DAYS 2014 ! CloudFront ! Elastic TranscoderHLS 24. 2424 JAWS DAYS 2014 25. 2525 JAWS DAYS 2014 ! ( ) ! ! AWS Amazon Simple Storage Service 26. 2626 JAWS DAYS 2014 ! ! ! 27. 2727 JAWS DAYS 2014 28. 2828 JAWS DAYS 2014 ! ! ! ! 29. 2929 JAWS DAYS 2014 ! APIHTML LogLog Log LogLog Log 30. 3030 JAWS DAYS 2014 CloudFront ! API ! ! 31. 3131 JAWS DAYS 2014 32. 3232 JAWS DAYS 2014 Just Released! ! CloudFront ! HTTP/HTTPS HTTP/HTTPS 33. 3333 JAWS DAYS 2014 Sumo Logic ! SaaS ! S3 ! 500MB/ ! 34. 3434 JAWS DAYS 2014 Splunk ! ! Splunk ! Market PlaceAMI ! http://blogs.splunk.com/2012/08/14/splunking-app-net/ 35. 3535 JAWS DAYS 2014 Cedexis ! ! ! 36. 3636 JAWS DAYS 2014 Google Analytics ! SaaS ! ! 1000/ http://www.google.co.jp/analytics/ 37. 3737 JAWS DAYS 2014 38. 3838 JAWS DAYS 2014 CloudFront ! S 3 ! S3 ! 34 ! HadoopRedshift 39. 3939 JAWS DAYS 2014 CloudFront ! PrexListObject s3DistCp ${DISTRIBUTION_NAME}-${YYYY}-${MM}-${DD}-${HH}.* 40. 4040 JAWS DAYS 2014 ! EMRS3DistCp SELECT COUNT(*) FROM. S3DistCp 41. 4141 JAWS DAYS 2014 ! EMR hive S3DistCPS3 hdfs:///dataconcat hdfs:///dataLocationExternal Table Hive 42. 4242 JAWS DAYS 2014 ! #EMR FLOW=`elastic-mapreduce --create --alive --hive-interactive --key-pair $ {KEYPAIR} --name emr-cluster-${TARGET} --master-instance-type $ {MASTER_INSTANCE_TYPE} --slave-instance-type $ {SLAVE_INSTANCE_TYPE} --num-instances ${NUM_INSTANCES} --log-uri ${LOG_URI} | awk '{print $4}'` #S3DistCp elastic-mapreduce --jobow $FLOW --jar /home/hadoop/lib/emr-s3distcp-1.0.jar --arg --s3Endpoint --arg "$S3ENDPOINT" --arg --src --arg "$CFLOG" --arg --dest --arg 'hdfs:///data' --arg --groupBy --arg '.*([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}).*' --arg --targetSize --arg '128' --arg --srcPattern --arg ".*${TARGET}.*" --arg --outputCodec --arg 'lzo' --step-name s3distcp ${TARGET}S3 128MB LZO hdfs:///data 43. 4343 JAWS DAYS 2014 ! ! Hive #Hive elastic-mapreduce --jobow $FLOW --hive-script $HIVESCRIPT --step-name table_construction CREATE EXTERNAL TABLE cflog ( dt STRING, tm STRING, edge STRING, bytes STRING, ip STRING, method STRING, host STRING, uri STRING, status STRING, referer STRING, ua STRING, querySTRING STRING, cookie STRING, resultType STRING, requestId STRING, hostHeader STRING, protocol STRING, csBytes STRING ) ROW FORMAT DELIMITED FIELDS TERMINATED BY 't' LOCATION '/data'; Hive 44. 4444 JAWS DAYS 2014 ! DynamoDB + Javascript SDK S3DistCp AWS SDK LINQ.JS D3.js EC2RDS1 45. 4545 JAWS DAYS 2014 ! DynamoDBSCAN D3.js 46. 4646 JAWS DAYS 2014 ! S3HDFS hive hiveDynamoDB 47. 4747 JAWS DAYS 2014 ! 48. 4848 JAWS DAYS 2014 ! DyamoDB 49. 4949 JAWS DAYS 2014 ! hive DynamoDB CREATE EXTERNAL TABLE dynamodb ( recordid STRING, ts STRING, uri STRING, edge STRING, count STRING ) STORED BY 'org.apache.hadoop.hive.dynamodb.DynamoDBStorageHandler' TBLPROPERTIES ( "dynamodb.table.name" = "cloudfront-digger", "dynamodb.region" = "ap-northeast-1", "dynamodb.throughput.write.percent" = "1.0", "dynamodb.throughput.read.percent" = "1.0", "dynamodb.column.mapping" = "recordid:Record_ID,ts:Timestamp,uri:URI,edge:Edge,count:Count" ); 50. 5050 JAWS DAYS 2014 ! DynamoDB INSERT OVERWRITE TABLE dynamodb SELECT concat(ts,'-',uri,'-',edge), ts, uri, edge, count(1) FROM source WHERE ts != 'NULL' GROUP BY ts, uri, edge ; 51. 5151 JAWS DAYS 2014 1. STS 2. DynamoDB1SCAN 3. LINQ.jsGroupByPivotCount 4. D3.js ! 52. 5252 JAWS DAYS 2014 ! 11,000EMRm1.small 13 53. 5353 JAWS DAYS 2014 BI ! Redshift + BI COPY 54. 5454 JAWS DAYS 2014 BI ! ! S3 CREATE TABLE cog ( recordid VARCHAR, ts CHAR, uri VARCHAR, edge CHAR, count INTEGER ) SORTKEY(ts); COPY cog FROM 's3://BUCKET_NAME/PREFIX CREDENTIALS 'aws_access_key_id=AAAAA;aws_secret_access_key=BBBB DELIMITER 't' MAXERROR 200; 55. 5555 JAWS DAYS 2014 ! BI ! ! ! CloudFront 56. 56 JAWS DAYS 2014