Related
I have written following code to stream data from Tweepy API. And I am getting data inside stream object. But unable to get streamp["user"]["followers_count"] but don't know how to get it. I also tried jsonLines = lines.flatMap(lambda json_str:json.loads(json_str)) but no help.
from __future__ import print_function
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
import json
sc = SparkContext()
ssc = StreamingContext(sc, 10) #Slide Interval of 10 sec
socket_stream = ssc.socketTextStream("localhost",4444)
stream = socket_stream.window(30) #window length 30 sec
stream.pprint()
ssc.start()
ssc.awaitTermination()
stream.pprint() gives me following JSON.
{"created_at":"Sat May 08 09:27:43 +0000 2021","id":1390961604079067137,"id_str":"1390961604079067137","text":"The return of the king! This style has so many parameters that have to be correct and Stone nails em all! - Drinkin\u2026 https:\/\/t.co\/uCogsfW8NC","source":"\u003ca href=\"https:\/\/untappd.com\" rel=\"nofollow\"\u003eUntappd\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":905439001,"id_str":"905439001","name":"Nahoj Morts","screen_name":"CraftBeerJunkie","location":null,"url":null,"description":null,"translator_type":"none","protected":false,"verified":false,"followers_count":15,"friends_count":113,"listed_count":0,"favourites_count":1,"statuses_count":1868,"created_at":"Fri Oct 26 06:36:01 +0000 2012","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/2765713047\/b92d58c569ad4739e67ef4d4e9a35780_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/2765713047\/b92d58c569ad4739e67ef4d4e9a35780_normal.jpeg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"The return of the king! This style has so many parameters that have to be correct and Stone nails em all! - Drinking a Stone Sublimely Self-Righteous Black IPA by #StoneBrewing # Uggleberget \u2014 https:\/\/t.co\/a3XixD5teo","display_text_range":[0,217],"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/a3XixD5teo","expanded_url":"https:\/\/untp.beer\/s\/c1025322433","display_url":"untp.beer\/s\/c1025322433","indices":[194,217]}],"user_mentions":[{"screen_name":"StoneBrewing","name":"Stone Brewing","id":16331259,"id_str":"16331259","indices":[163,176]}],"symbols":[]}},"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/uCogsfW8NC","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390961604079067137","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1620466063886"}
{"created_at":"Sat May 08 09:27:43 +0000 2021","id":1390961604334919683,"id_str":"1390961604334919683","text":"[HQ] 210508 #KrisWu #Wuyifan # GTSSC 2021 (Day 1) Cr.Fanbaobao(3) #PorscheRacerKrisWu \nhttps:\/\/t.co\/cEcAtzKNsM\u2026 https:\/\/t.co\/3B9uUS3mdu","display_text_range":[0,140],"source":"\u003ca href=\"https:\/\/mobile.twitter.com\" rel=\"nofollow\"\u003eTwitter Web App\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":1390961304316387331,"in_reply_to_status_id_str":"1390961304316387331","in_reply_to_user_id":620222301,"in_reply_to_user_id_str":"620222301","in_reply_to_screen_name":"kissmemyfan_","user":{"id":620222301,"id_str":"620222301","name":"KISSMEMYFAN1106\u2661","screen_name":"kissmemyfan_","location":null,"url":"https:\/\/www.youtube.com\/channel\/UCtYrDyYwqZyQfYtCz79zPHQ\/featured","description":"for #KrisWu Kris Wu Yifan since 120627 \u2740notify in case of emergency,pls dm\u2740 fb: https:\/\/www.facebook.com\/groups\/1051879991503131\/","translator_type":"regular","protected":false,"verified":false,"followers_count":91307,"friends_count":24,"listed_count":433,"favourites_count":14194,"statuses_count":95476,"created_at":"Wed Jun 27 16:34:02 +0000 2012","utc_offset":null,"time_zone":null,"geo_enabled":true,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"FFCC4D","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":true,"profile_link_color":"E81C4F","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1252957871802707974\/RjNGNnly_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1252957871802707974\/RjNGNnly_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/620222301\/1587563426","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"[HQ] 210508 #KrisWu #Wuyifan # GTSSC 2021 (Day 1) Cr.Fanbaobao(3) #PorscheRacerKrisWu \nhttps:\/\/t.co\/cEcAtzKNsM\nhttps:\/\/t.co\/PNbPqE88Y3 https:\/\/t.co\/8WPhfpHiyO","display_text_range":[0,134],"entities":{"hashtags":[{"text":"KrisWu","indices":[12,19]},{"text":"Wuyifan","indices":[20,28]},{"text":"PorscheRacerKrisWu","indices":[66,85]}],"urls":[{"url":"https:\/\/t.co\/cEcAtzKNsM","expanded_url":"https:\/\/wx1.sinaimg.cn\/large\/00688bwvly1gqb3c1cdbuj31l82bpe81.jpg","display_url":"wx1.sinaimg.cn\/large\/00688bwv\u2026","indices":[87,110]},{"url":"https:\/\/t.co\/PNbPqE88Y3","expanded_url":"https:\/\/wx4.sinaimg.cn\/large\/00688bwvly1gqb3c1mc9hj31ay1wb7ji.jpg","display_url":"wx4.sinaimg.cn\/large\/00688bwv\u2026","indices":[111,134]}],"user_mentions":[],"symbols":[],"media":[{"id":1390961423400857602,"id_str":"1390961423400857602","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"small":{"w":465,"h":680,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":1024,"h":1498,"resize":"fit"},"medium":{"w":820,"h":1200,"resize":"fit"}}},{"id":1390961423417704450,"id_str":"1390961423417704450","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":825,"h":1200,"resize":"fit"},"small":{"w":467,"h":680,"resize":"fit"},"large":{"w":1024,"h":1490,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390961423400857602,"id_str":"1390961423400857602","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"small":{"w":465,"h":680,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":1024,"h":1498,"resize":"fit"},"medium":{"w":820,"h":1200,"resize":"fit"}}},{"id":1390961423417704450,"id_str":"1390961423417704450","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":825,"h":1200,"resize":"fit"},"small":{"w":467,"h":680,"resize":"fit"},"large":{"w":1024,"h":1490,"resize":"fit"}}}]}},"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"KrisWu","indices":[12,19]},{"text":"Wuyifan","indices":[20,28]},{"text":"PorscheRacerKrisWu","indices":[66,85]}],"urls":[{"url":"https:\/\/t.co\/cEcAtzKNsM","expanded_url":"https:\/\/wx1.sinaimg.cn\/large\/00688bwvly1gqb3c1cdbuj31l82bpe81.jpg","display_url":"wx1.sinaimg.cn\/large\/00688bwv\u2026","indices":[87,110]},{"url":"https:\/\/t.co\/3B9uUS3mdu","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390961604334919683","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[112,135]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ht","timestamp_ms":"1620466063947"}
{"created_at":"Sat May 08 09:27:44 +0000 2021","id":1390961607572865031,"id_str":"1390961607572865031","text":"RT #aa86marat2: #\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u0647\u0627\n\u0633\u062c\u064a\u0646\u2026","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1250758860660527105,"id_str":"1250758860660527105","name":"om_mhamd99","screen_name":"Mhamd99Om","location":null,"url":null,"description":"\u064a\u0627\u0631\u0628 \u2728\ud83e\udd32\ud83c\udffb\u0628\u0627\u0631\u0643 \u0644\u064a \u0641\u064a\u0645\u0627 \u0623\u0639\u0637\u062a\u0646\u064a \u0648\u0641\u0631\u062c \u0647\u0645\u064a \u0628\u0642\u0636\u0627\u0621 \u062f\u064a\u0646 \u0648\u0627\u0639\u062a\u064a \u0639\u0644\u064a \u0630\u0644\u0643 \u0641\u0627\u062a\u0648\u0631\u0647 \u062a\u0646\u0641\u064a\u0630 1935092757","translator_type":"none","protected":false,"verified":false,"followers_count":903,"friends_count":389,"listed_count":0,"favourites_count":2174,"statuses_count":33294,"created_at":"Thu Apr 16 12:12:15 +0000 2020","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1286520143431061504\/k2Jg3W0u_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1286520143431061504\/k2Jg3W0u_normal.jpg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sat May 08 02:27:00 +0000 2021","id":1390855727116427270,"id_str":"1390855727116427270","text":"#\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u2026 https:\/\/t.co\/mpCKrtjMW4","display_text_range":[0,140],"source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1338279194892115974,"id_str":"1338279194892115974","name":"\u0639\u0628\u062f\u0627\u0644\u0631\u062d\u0645\u0646 \u0645\u062d\u0645\u062f 2","screen_name":"aa86marat2","location":null,"url":null,"description":null,"translator_type":"none","protected":false,"verified":false,"followers_count":382,"friends_count":82,"listed_count":1,"favourites_count":43,"statuses_count":16494,"created_at":"Mon Dec 14 00:26:54 +0000 2020","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1356322484673519616\/7NSfbGye_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1356322484673519616\/7NSfbGye_normal.jpg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"#\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u0647\u0627\n\u0633\u062c\u064a\u0646 \u0639\u0645\u0631\u064756 \u0639\u0627\u0645\u0627 \u0645\u062a\u0632\u0648\u062c \u0644\u062f\u064a\u0647 \u0637\u0641\u0644 \u0645\u0633\u062c\u0648\u0646 \u0645\u0646\u0630\u0639\u0627\u0645\u064a\u0646 \u06484\u0623\u0634\u0647\u0631\n \u0645\u062a\u0628\u0642\u0649 \u0639\u0644\u064a\u0647 296200 \u0631\u064a\u0627\u0644\n\u0627\u0644\u0641\u0627\u062a\u0648\u0631\u0629\n1934638011\n\u0639\u0628\u0631 #\n\nhttps:\/\/t.co\/JjOHPoGLrJ\n\n\u0627\u0644\u062c\u0631\u0623\u0629_\u0641\u064a_\u0627\u0644\u0637\u0628\u0639\n\u0645\u0635\u0631\u0641_\u0627\u0644\u0631\u0627\u062c\u062d\u064a https:\/\/t.co\/mGuOo9qwZP","display_text_range":[0,276],"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[0,20]}],"urls":[{"url":"https:\/\/t.co\/JjOHPoGLrJ","expanded_url":"https:\/\/Ehsan.sa\/referral\/29734F898E5CCB199DD92FD5CE8284C3B10081046BB13791AC72608BDD62B3D84020AC2B515E485C2E4B718A425C1A710438C7B814161E4197F6E7F4F73557AA","display_url":"Ehsan.sa\/referral\/29734\u2026","indices":[223,246]}],"user_mentions":[],"symbols":[],"media":[{"id":1390855708363698183,"id_str":"1390855708363698183","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":554,"h":1200,"resize":"fit"},"large":{"w":720,"h":1560,"resize":"fit"},"small":{"w":314,"h":680,"resize":"fit"}}},{"id":1390855718069317649,"id_str":"1390855718069317649","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":511,"h":680,"resize":"fit"},"large":{"w":901,"h":1200,"resize":"fit"},"medium":{"w":901,"h":1200,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390855708363698183,"id_str":"1390855708363698183","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":554,"h":1200,"resize":"fit"},"large":{"w":720,"h":1560,"resize":"fit"},"small":{"w":314,"h":680,"resize":"fit"}}},{"id":1390855718069317649,"id_str":"1390855718069317649","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":511,"h":680,"resize":"fit"},"large":{"w":901,"h":1200,"resize":"fit"},"medium":{"w":901,"h":1200,"resize":"fit"}}}]}},"quote_count":0,"reply_count":1,"retweet_count":33,"favorite_count":2,"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[0,20]}],"urls":[{"url":"https:\/\/t.co\/mpCKrtjMW4","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390855727116427270","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ar"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[16,36]}],"urls":[],"user_mentions":[{"screen_name":"aa86marat2","name":"\u0639\u0628\u062f\u0627\u0644\u0631\u062d\u0645\u0646 \u0645\u062d\u0645\u062f 2","id":1338279194892115974,"id_str":"1338279194892115974","indices":[3,14]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"ar","timestamp_ms":"1620466064719"}
{"created_at":"Sat May 08 09:27:44 +0000 2021","id":1390961607799296000,"id_str":"1390961607799296000","text":"RT #ariftgif: Tag ur bestie \/ fav person \/ # third person https:\/\/t.co\/L4pGn5nAKJ","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":812488479607169024,"id_str":"812488479607169024","name":"\ud83c\udfae","screen_name":"bjeyyy_","location":null,"url":null,"description":"always 20 hihihi","translator_type":"none","protected":false,"verified":false,"followers_count":100,"friends_count":108,"listed_count":0,"favourites_count":2519,"statuses_count":48348,"created_at":"Sat Dec 24 02:42:03 +0000 2016","utc_offset":null,"time_zone":null,"geo_enabled":true,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1381271588394336256\/CnmfqTCN_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1381271588394336256\/CnmfqTCN_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/812488479607169024\/1582382619","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Fri May 07 10:38:58 +0000 2021","id":1390617147198480388,"id_str":"1390617147198480388","text":"Tag ur bestie \/ fav person \/ # third person https:\/\/t.co\/L4pGn5nAKJ","display_text_range":[0,43],"source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1188902968277385216,"id_str":"1188902968277385216","name":"arif | \u2606","screen_name":"ariftgif","location":"\ud835\udcb6\ud835\udcc3\ud835\udcbf\ud835\udcb6\ud835\udcc8\ud835\udcc2\ud835\udcb6\ud835\udcc7\ud835\udcb6","url":"https:\/\/vt.tiktok.com\/ZSEAw2WN\/","description":"ugly asf | a man with mullet hair","translator_type":"none","protected":false,"verified":false,"followers_count":2144,"friends_count":937,"listed_count":0,"favourites_count":20934,"statuses_count":19470,"created_at":"Mon Oct 28 19:39:09 +0000 2019","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1387751035041505282\/f_Q5rJ_B_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1387751035041505282\/f_Q5rJ_B_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1188902968277385216\/1619601431","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"quote_count":49,"reply_count":176,"retweet_count":1451,"favorite_count":3773,"entities":{"hashtags":[],"urls":[],"user_mentions":[],"symbols":[],"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[44,67],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[44,67],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}}}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"ariftgif","name":"arif | \u2606","id":1188902968277385216,"id_str":"1188902968277385216","indices":[3,12]}],"symbols":[],"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[58,81],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}},"source_status_id":1390617147198480388,"source_status_id_str":"1390617147198480388","source_user_id":1188902968277385216,"source_user_id_str":"1188902968277385216"}]},"extended_entities":{"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[58,81],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}},"source_status_id":1390617147198480388,"source_status_id_str":"1390617147198480388","source_user_id":1188902968277385216,"source_user_id_str":"1188902968277385216"}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1620466064773"}
I am new to spark and already spent 3 days finding out how to get value from Nested JSON Inside DStream Object.
You can map the json string to a tuple of the values that you want to extract:
import json
def parse_json_string(str):
user = json.loads(str)["user"]
return (user["name"], user["followers_count"])
stream = socket_stream.window...
stream = stream.map(lambda str: parse_json_string(str))
stream.pprint()
ssc.start()
ssc.awaitTermination()
Output:
-------------------------------------------
Time: 2021-05-08 18:50:48
-------------------------------------------
('Nahoj Morts', 15)
('KISSMEMYFAN1106♡', 91307)
('om_mhamd99', 903)
('🎮', 100)
I'm web scraping a HTML page which shows a time series generated by a function inside a <script> tag. On the webbpage I can see different dates for the time series, but when I "view page source" these dates seem to be converted into numbers.
For example "1615856400000" = "Thuesday, Mar 16, 2021", "1615770000" = "Monday, Mar 15, 2021" and "1612227600000" = "Thuesday, Feb 2, 2021".
Does anyone know what the logic is for these conversions? Ideally I would like a list which shows the mappings per day, starting at some arbitrary date e.g. "Wednesday, Feb 26, 2014".
What I'm trying to do here is to leave texts only from each tweet.
import org.apache.spark.{SparkConf, SparkContext}
import scala.io.Source
object shortTwitter {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("ShortTwitterAnalysis").setMaster("local[2]")
val sc = new SparkContext(sparkConf)
val text = sc.textFile("/home/tobby/data/shortTwitter.txt")
val counts = text
.map(_.toLowerCase)
.map(_.toString)
.map(_.replace("\t", ""))
.map(_.replace("\"", ""))
.map(_.replace("\n", ""))
.map(_.replaceAll("[\\p{C}]", ""))
.map(_.split("\"text\":\"")(1).split("\",\"source\":")(0))
counts.foreach(println)
}
}
But the last map function .map(_.split("\"text\":\"")(1).split("\",\"source\":")(0)) does not work. Do you have any advice?
Without the .map(_.split("\"text\":\"")(1).split("\",\"source\":")(0)) my tweets look like below :
{created_at:wed jul 16 23:58:19 +0000 2014,id:489559687189110784,id_str:489559687189110784,text:a rose by any other name would smell as sweet,source:\u003ca href=\https:\/\/twitter.com\/download\/android\ rel=\nofollow\\u003etwitter for android\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:621244372,id_str:621244372,name:\u2665,screen_name:ivunia_ontrinae,location:,url:null,description:me myself & i \u2764,protected:false,verified:false,followers_count:1023,friends_count:591,listed_count:1,favourites_count:1909,statuses_count:26770,created_at:thu jun 28 19:23:06 +0000 2012,utc_offset:-10800,time_zone:atlantic time (canada),geo_enabled:true,lang:en,contributors_enabled:false,is_translator:false,profile_background_color:c0deed,profile_background_image_url:http:\/\/pbs.twimg.com\/profile_background_images\/378800000101658269\/ec0820565f0451a3ce7169c776fbe41f.jpeg,profile_background_image_url_https:https:\/\/pbs.twimg.com\/profile_background_images\/378800000101658269\/ec0820565f0451a3ce7169c776fbe41f.jpeg,profile_background_tile:true,profile_link_color:e62bb4,profile_sidebar_border_color:000000,profile_sidebar_fill_color:ddeef6,profile_text_color:333333,profile_use_background_image:true,profile_image_url:http:\/\/pbs.twimg.com\/profile_images\/483373612749959168\/f3qpy_66_normal.jpeg,profile_image_url_https:https:\/\/pbs.twimg.com\/profile_images\/483373612749959168\/f3qpy_66_normal.jpeg,profile_banner_url:https:\/\/pbs.twimg.com\/profile_banners\/621244372\/1404758956,default_profile:false,default_profile_image:false,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,retweet_count:0,favorite_count:0,entities:{hashtags:[],trends:[],urls:[],user_mentions:[],symbols:[]},favorited:false,retweeted:false,possibly_sensitive:false,filter_level:medium,lang:en}
{created_at:wed jul 16 23:58:19 +0000 2014,id:489559687189110784,id_str:489559687189110784,text:a rose is a rose is a rose,source:\u003ca href=\https:\/\/twitter.com\/download\/android\ rel=\nofollow\\u003etwitter for android\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:621244372,id_str:621244372,name:\u2665,screen_name:ivunia_ontrinae,location:,url:null,description:me myself & i \u2764,protected:false,verified:false,followers_count:1023,friends_count:591,listed_count:1,favourites_count:1909,statuses_count:26770,created_at:thu jun 28 19:23:06 +0000 2012,utc_offset:-10800,time_zone:atlantic time (canada),geo_enabled:true,lang:en,contributors_enabled:false,is_translator:false,profile_background_color:c0deed,profile_background_image_url:http:\/\/pbs.twimg.com\/profile_background_images\/378800000101658269\/ec0820565f0451a3ce7169c776fbe41f.jpeg,profile_background_image_url_https:https:\/\/pbs.twimg.com\/profile_background_images\/378800000101658269\/ec0820565f0451a3ce7169c776fbe41f.jpeg,profile_background_tile:true,profile_link_color:e62bb4,profile_sidebar_border_color:000000,profile_sidebar_fill_color:ddeef6,profile_text_color:333333,profile_use_background_image:true,profile_image_url:http:\/\/pbs.twimg.com\/profile_images\/483373612749959168\/f3qpy_66_normal.jpeg,profile_image_url_https:https:\/\/pbs.twimg.com\/profile_images\/483373612749959168\/f3qpy_66_normal.jpeg,profile_banner_url:https:\/\/pbs.twimg.com\/profile_banners\/621244372\/1404758956,default_profile:false,default_profile_image:false,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,retweet_count:0,favorite_count:0,entities:{hashtags:[],trends:[],urls:[],user_mentions:[],symbols:[]},favorited:false,retweeted:false,possibly_sensitive:false,filter_level:medium,lang:en}
{created_at:wed jul 16 23:58:19 +0000 2014,id:489559687176945664,id_str:489559687176945664,text:love is like a rose the joy of all the earth,source:\u003ca href=\http:\/\/twitter.com\/download\/iphone\ rel=\nofollow\\u003etwitter for iphone\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:363819213,id_str:363819213,name:ivanna010394,screen_name:ivannacarrillo,location:,url:null,description:null,protected:false,verified:false,followers_count:243,friends_count:530,listed_count:0,favourites_count:26,statuses_count:5672,created_at:sun aug 28 18:58:49 +0000 2011,utc_offset:-14400,time_zone:eastern time (us & canada),geo_enabled:false,lang:es,contributors_enabled:false,is_translator:false,profile_background_color:642d8b,profile_background_image_url:http:\/\/pbs.twimg.com\/profile_background_images\/767201253\/661eb2d4915e9ee6566647dcbaab0186.jpeg,profile_background_image_url_https:https:\/\/pbs.twimg.com\/profile_background_images\/767201253\/661eb2d4915e9ee6566647dcbaab0186.jpeg,profile_background_tile:true,profile_link_color:ff0000,profile_sidebar_border_color:ffffff,profile_sidebar_fill_color:7ac3ee,profile_text_color:3d1957,profile_use_background_image:true,profile_image_url:http:\/\/pbs.twimg.com\/profile_images\/455873054703648768\/_b4mf6o7_normal.jpeg,profile_image_url_https:https:\/\/pbs.twimg.com\/profile_images\/455873054703648768\/_b4mf6o7_normal.jpeg,profile_banner_url:https:\/\/pbs.twimg.com\/profile_banners\/363819213\/1402261141,default_profile:false,default_profile_image:false,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,retweeted_status:{created_at:wed jul 16 13:45:28 +0000 2014,id:489405458168709120,id_str:489405458168709120,text:our milan show is now sold out, thankyou :d tickets are still available for most of europe ! http:\/\/t.co\/arnh7pvoap http:\/\/t.co\/t5wzyocrtu,source:\u003ca href=\http:\/\/twitter.com\ rel=\nofollow\\u003etwitter web client\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:264107729,id_str:264107729,name:5 seconds of summer,screen_name:5sos,location:sydney, australia,url:http:\/\/www.facebook.com\/5secondsofsummer,description:4 aussies making music :) love the people who support us! our album is out :) http:\/\/po.st\/or93y4 | #ashton5sos #calum5sos #michael5sos #luke5sos,protected:false,verified:true,followers_count:3704204,friends_count:28660,listed_count:20024,favourites_count:1061,statuses_count:17297,created_at:fri mar 11 10:18:46 +0000 2011,utc_offset:36000,time_zone:sydney,geo_enabled:false,lang:en,contributors_enabled:false,is_translator:false,profile_background_color:000000,profile_background_image_url:http:\/\/pbs.twimg.com\/profile_background_images\/483531430371147778\/0gzkh2zi.jpeg,profile_background_image_url_https:https:\/\/pbs.twimg.com\/profile_background_images\/483531430371147778\/0gzkh2zi.jpeg,profile_background_tile:false,profile_link_color:c21b1b,profile_sidebar_border_color:ffffff,profile_sidebar_fill_color:ddeef6,profile_text_color:333333,profile_use_background_image:true,profile_image_url:http:\/\/pbs.twimg.com\/profile_images\/485730748574752768\/zm1ctcvv_normal.jpeg,profile_image_url_https:https:\/\/pbs.twimg.com\/profile_images\/485730748574752768\/zm1ctcvv_normal.jpeg,profile_banner_url:https:\/\/pbs.twimg.com\/profile_banners\/264107729\/1404117825,default_profile:false,default_profile_image:false,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,retweet_count:12648,favorite_count:31390,entities:{hashtags:[],trends:[],urls:[{url:http:\/\/t.co\/arnh7pvoap,expanded_url:http:\/\/5sos.com\/live,display_url:5sos.com\/live,indices:[93,115]}],user_mentions:[],symbols:[],media:[{id:489405457111715840,id_str:489405457111715840,indices:[116,138],media_url:http:\/\/pbs.twimg.com\/media\/bsq3q5zieaakbgg.jpg,media_url_https:https:\/\/pbs.twimg.com\/media\/bsq3q5zieaakbgg.jpg,url:http:\/\/t.co\/t5wzyocrtu,display_url:pic.twitter.com\/t5wzyocrtu,expanded_url:http:\/\/twitter.com\/5sos\/status\/489405458168709120\/photo\/1,type:photo,sizes:{small:{w:340,h:613,resize:fit},thumb:{w:150,h:150,resize:crop},medium:{w:600,h:1081,resize:fit},large:{w:811,h:1461,resize:fit}}}]},favorited:false,retweeted:false,possibly_sensitive:false,filter_level:low,lang:en},retweet_count:0,favorite_count:0,entities:{hashtags:[],trends:[],urls:[{url:http:\/\/t.co\/arnh7pvoap,expanded_url:http:\/\/5sos.com\/live,display_url:5sos.com\/live,indices:[103,125]}],user_mentions:[{screen_name:5sos,name:5 seconds of summer,id:264107729,id_str:264107729,indices:[3,8]}],symbols:[],media:[{id:489405457111715840,id_str:489405457111715840,indices:[126,140],media_url:http:\/\/pbs.twimg.com\/media\/bsq3q5zieaakbgg.jpg,media_url_https:https:\/\/pbs.twimg.com\/media\/bsq3q5zieaakbgg.jpg,url:http:\/\/t.co\/t5wzyocrtu,display_url:pic.twitter.com\/t5wzyocrtu,expanded_url:http:\/\/twitter.com\/5sos\/status\/489405458168709120\/photo\/1,type:photo,sizes:{small:{w:340,h:613,resize:fit},thumb:{w:150,h:150,resize:crop},medium:{w:600,h:1081,resize:fit},large:{w:811,h:1461,resize:fit}},source_status_id:489405458168709120,source_status_id_str:489405458168709120}]},favorited:false,retweeted:false,possibly_sensitive:false,filter_level:medium,lang:en}
{created_at:sat jan 16 12:00:47 +0000 2016,id:688330052233199616,id_str:688330052233199616,text:rt #nba2k: the battle of two young teams. tough season but one will emerge victorious. who will it be? lakers or 76ers? https:\/\/t.co\/nukkjq\u2026,source:\u003ca href=\http:\/\/twitter.com\ rel=\nofollow\\u003etwitter web client\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:4817727209,id_str:4817727209,name:mark lieyg,screen_name:_yungwiggins_,location:null,url:null,description:null,protected:false,verified:false,followers_count:3,friends_count:40,listed_count:0,favourites_count:0,statuses_count:39,created_at:sat jan 16 11:06:38 +0000 2016,utc_offset:-28800,time_zone:pacific time (us & canada),geo_enabled:false,lang:en,contributors_enabled:false,is_translator:false,profile_background_color:f5f8fa,profile_background_image_url:,profile_background_image_url_https:,profile_background_tile:false,profile_link_color:2b7bb9,profile_sidebar_border_color:c0deed,profile_sidebar_fill_color:ddeef6,profile_text_color:333333,profile_use_background_image:true,profile_image_url:http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_1_normal.png,profile_image_url_https:https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_1_normal.png,default_profile:true,default_profile_image:true,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,retweeted_status: {created_at:sat jan 02 03:31:10 +0000 2016,id:683128371627200513,id_str:683128371627200513,text:the battle of two young teams. tough season but one will emerge victorious. who will it be? lakers or 76ers? https:\/\/t.co\/nukkjqqspa,source:\u003ca href=\http:\/\/percolate.com\ rel=\nofollow\\u003epercolate\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:15573174,id_str:15573174,name:nba 2k 2k16,screen_name:nba2k,location:novato, ca,url:http:\/\/www.2k.com,description:esrb rating: everyone 10+. #nba2k16 available now for playstation 4 & xbox one, playstation 3 & xbox 360 & pc http:\/\/2kgam.es\/buynba2k16,protected:false,verified:true,followers_count:948071,friends_count:1630,listed_count:3305,favourites_count:10,statuses_count:8162,created_at:wed jul 23 21:57:14 +0000 2008,utc_offset:-28800,time_zone:pacific time (us & canada),geo_enabled:true,lang:en,contributors_enabled:false,is_translator:false,profile_background_color:000000,profile_background_image_url:http:\/\/pbs.twimg.com\/profile_background_images\/539865904528371712\/gnb-ggrq.png,profile_background_image_url_https:https:\/\/pbs.twimg.com\/profile_background_images\/539865904528371712\/gnb-ggrq.png,profile_background_tile:false,profile_link_color:ff0300,profile_sidebar_border_color:ffffff,profile_sidebar_fill_color:0d2b44,profile_text_color:408af2,profile_use_background_image:true,profile_image_url:http:\/\/pbs.twimg.com\/profile_images\/606562975109890048\/sumjozun_normal.jpg,profile_image_url_https:https:\/\/pbs.twimg.com\/profile_images\/606562975109890048\/sumjozun_normal.jpg,profile_banner_url:https:\/\/pbs.twimg.com\/profile_banners\/15573174\/1433457451,default_profile:false,default_profile_image:false,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,is_quote_status:false,retweet_count:112,favorite_count:547,entities:{hashtags:[],urls:[],user_mentions:[],symbols:[],media:[{id:683128370796736512,id_str:683128370796736512,indices:[109,132],media_url:http:\/\/pbs.twimg.com\/media\/cxr1okvusaamnu4.jpg,media_url_https:https:\/\/pbs.twimg.com\/media\/cxr1okvusaamnu4.jpg,url:https:\/\/t.co\/nukkjqqspa,display_url:pic.twitter.com\/nukkjqqspa,expanded_url:http:\/\/twitter.com\/nba2k\/status\/683128371627200513\/photo\/1,type:photo,sizes:{large:{w:1024,h:419,resize:fit},thumb:{w:150,h:150,resize:crop},medium:{w:600,h:245,resize:fit},small:{w:340,h:139,resize:fit}}}]},extended_entities:{media:[{id:683128370796736512,id_str:683128370796736512,indices:[109,132],media_url:http:\/\/pbs.twimg.com\/media\/cxr1okvusaamnu4.jpg,media_url_https:https:\/\/pbs.twimg.com\/media\/cxr1okvusaamnu4.jpg,url:https:\/\/t.co\/nukkjqqspa,display_url:pic.twitter.com\/nukkjqqspa,expanded_url:http:\/\/twitter.com\/nba2k\/status\/683128371627200513\/photo\/1,type:photo,sizes:{large:{w:1024,h:419,resize:fit},thumb:{w:150,h:150,resize:crop},medium:{w:600,h:245,resize:fit},small:{w:340,h:139,resize:fit}}}]},favorited:false,retweeted:false,possibly_sensitive:false,filter_level:low,lang:en},is_quote_status:false,retweet_count:0,favorite_count:0,entities:{hashtags:[],urls:[],user_mentions:[{screen_name:nba2k,name:nba 2k 2k16,id:15573174,id_str:15573174,indices:[3,9]}],symbols:[],media:[{id:683128370796736512,id_str:683128370796736512,indices:[120,140],media_url:http:\/\/pbs.twimg.com\/media\/cxr1okvusaamnu4.jpg,media_url_https:https:\/\/pbs.twimg.com\/media\/cxr1okvusaamnu4.jpg,url:https:\/\/t.co\/nukkjqqspa,display_url:pic.twitter.com\/nukkjqqspa,expanded_url:http:\/\/twitter.com\/nba2k\/status\/683128371627200513\/photo\/1,type:photo,sizes:{large:{w:1024,h:419,resize:fit},thumb:{w:150,h:150,resize:crop},medium:{w:600,h:245,resize:fit},small:{w:340,h:139,resize:fit}},source_status_id:683128371627200513,source_status_id_str:683128371627200513,source_user_id:15573174,source_user_id_str:15573174}]},extended_entities:{media:[{id:683128370796736512,id_str:683128370796736512,indices:[120,140],media_url:http:\/\/pbs.twimg.com\/media\/cxr1okvusaamnu4.jpg,media_url_https:https:\/\/pbs.twimg.com\/media\/cxr1okvusaamnu4.jpg,url:https:\/\/t.co\/nukkjqqspa,display_url:pic.twitter.com\/nukkjqqspa,expanded_url:http:\/\/twitter.com\/nba2k\/status\/683128371627200513\/photo\/1,type:photo,sizes:{large:{w:1024,h:419,resize:fit},thumb:{w:150,h:150,resize:crop},medium:{w:600,h:245,resize:fit},small:{w:340,h:139,resize:fit}},source_status_id:683128371627200513,source_status_id_str:683128371627200513,source_user_id:15573174,source_user_id_str:15573174}]},favorited:false,retweeted:false,possibly_sensitive:false,filter_level:low,lang:en,timestamp_ms:1452945647663}
{created_at:wed jul 16 23:58:19 +0000 2014,id:489559687176945664,id_str:489559687176945664,text:at christmas i no more desire a rose than wish a snow in may’s new-fangled mirth,source:\u003ca href=\http:\/\/twitter.com\/download\/iphone\ rel=\nofollow\\u003etwitter for iphone\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:363819213,id_str:363819213,name:ivanna010394,screen_name:ivannacarrillo,location:,url:null,description:null,protected:false,verified:false,followers_count:243,friends_count:530,listed_count:0,favourites_count:26,statuses_count:5672,created_at:sun aug 28 18:58:49 +0000 2011,utc_offset:-14400,time_zone:eastern time (us & canada),geo_enabled:false,lang:es,contributors_enabled:false,is_translator:false,profile_background_color:642d8b,profile_background_image_url:http:\/\/pbs.twimg.com\/profile_background_images\/767201253\/661eb2d4915e9ee6566647dcbaab0186.jpeg,profile_background_image_url_https:https:\/\/pbs.twimg.com\/profile_background_images\/767201253\/661eb2d4915e9ee6566647dcbaab0186.jpeg,profile_background_tile:true,profile_link_color:ff0000,profile_sidebar_border_color:ffffff,profile_sidebar_fill_color:7ac3ee,profile_text_color:3d1957,profile_use_background_image:true,profile_image_url:http:\/\/pbs.twimg.com\/profile_images\/455873054703648768\/_b4mf6o7_normal.jpeg,profile_image_url_https:https:\/\/pbs.twimg.com\/profile_images\/455873054703648768\/_b4mf6o7_normal.jpeg,profile_banner_url:https:\/\/pbs.twimg.com\/profile_banners\/363819213\/1402261141,default_profile:false,default_profile_image:false,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,retweeted_status:{created_at:wed jul 16 13:45:28 +0000 2014,id:489405458168709120,id_str:489405458168709120,text:our milan show is now sold out, thankyou :d tickets are still available for most of europe ! http:\/\/t.co\/arnh7pvoap http:\/\/t.co\/t5wzyocrtu,source:\u003ca href=\http:\/\/twitter.com\ rel=\nofollow\\u003etwitter web client\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:264107729,id_str:264107729,name:5 seconds of summer,screen_name:5sos,location:sydney, australia,url:http:\/\/www.facebook.com\/5secondsofsummer,description:4 aussies making music :) love the people who support us! our album is out :) http:\/\/po.st\/or93y4 | #ashton5sos #calum5sos #michael5sos #luke5sos,protected:false,verified:true,followers_count:3704204,friends_count:28660,listed_count:20024,favourites_count:1061,statuses_count:17297,created_at:fri mar 11 10:18:46 +0000 2011,utc_offset:36000,time_zone:sydney,geo_enabled:false,lang:en,contributors_enabled:false,is_translator:false,profile_background_color:000000,profile_background_image_url:http:\/\/pbs.twimg.com\/profile_background_images\/483531430371147778\/0gzkh2zi.jpeg,profile_background_image_url_https:https:\/\/pbs.twimg.com\/profile_background_images\/483531430371147778\/0gzkh2zi.jpeg,profile_background_tile:false,profile_link_color:c21b1b,profile_sidebar_border_color:ffffff,profile_sidebar_fill_color:ddeef6,profile_text_color:333333,profile_use_background_image:true,profile_image_url:http:\/\/pbs.twimg.com\/profile_images\/485730748574752768\/zm1ctcvv_normal.jpeg,profile_image_url_https:https:\/\/pbs.twimg.com\/profile_images\/485730748574752768\/zm1ctcvv_normal.jpeg,profile_banner_url:https:\/\/pbs.twimg.com\/profile_banners\/264107729\/1404117825,default_profile:false,default_profile_image:false,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,retweet_count:12648,favorite_count:31390,entities:{hashtags:[],trends:[],urls:[{url:http:\/\/t.co\/arnh7pvoap,expanded_url:http:\/\/5sos.com\/live,display_url:5sos.com\/live,indices:[93,115]}],user_mentions:[],symbols:[],media:[{id:489405457111715840,id_str:489405457111715840,indices:[116,138],media_url:http:\/\/pbs.twimg.com\/media\/bsq3q5zieaakbgg.jpg,media_url_https:https:\/\/pbs.twimg.com\/media\/bsq3q5zieaakbgg.jpg,url:http:\/\/t.co\/t5wzyocrtu,display_url:pic.twitter.com\/t5wzyocrtu,expanded_url:http:\/\/twitter.com\/5sos\/status\/489405458168709120\/photo\/1,type:photo,sizes:{small:{w:340,h:613,resize:fit},thumb:{w:150,h:150,resize:crop},medium:{w:600,h:1081,resize:fit},large:{w:811,h:1461,resize:fit}}}]},favorited:false,retweeted:false,possibly_sensitive:false,filter_level:low,lang:en},retweet_count:0,favorite_count:0,entities:{hashtags:[],trends:[],urls:[{url:http:\/\/t.co\/arnh7pvoap,expanded_url:http:\/\/5sos.com\/live,display_url:5sos.com\/live,indices:[103,125]}],user_mentions:[{screen_name:5sos,name:5 seconds of summer,id:264107729,id_str:264107729,indices:[3,8]}],symbols:[],media:[{id:489405457111715840,id_str:489405457111715840,indices:[126,140],media_url:http:\/\/pbs.twimg.com\/media\/bsq3q5zieaakbgg.jpg,media_url_https:https:\/\/pbs.twimg.com\/media\/bsq3q5zieaakbgg.jpg,url:http:\/\/t.co\/t5wzyocrtu,display_url:pic.twitter.com\/t5wzyocrtu,expanded_url:http:\/\/twitter.com\/5sos\/status\/489405458168709120\/photo\/1,type:photo,sizes:{small:{w:340,h:613,resize:fit},thumb:{w:150,h:150,resize:crop},medium:{w:600,h:1081,resize:fit},large:{w:811,h:1461,resize:fit}},source_status_id:489405458168709120,source_status_id_str:489405458168709120}]},favorited:false,retweeted:false,possibly_sensitive:false,filter_level:medium,lang:en}
{created_at:sat jan 16 12:00:48 +0000 2016,id:688330056410755072,id_str:688330056410755072,text:i was going to bake a cake and listen to the football. flour refund?,source:\u003ca href=\http:\/\/twitter.com\/download\/iphone\ rel=\nofollow\\u003etwitter for iphone\u003c\/a\u003e,truncated:false,in_reply_to_status_id:null,in_reply_to_status_id_str:null,in_reply_to_user_id:null,in_reply_to_user_id_str:null,in_reply_to_screen_name:null,user:{id:252303653,id_str:252303653,name:pete blackman,screen_name:peteblackman,location:null,url:null,description:null,protected:false,verified:false,followers_count:409,friends_count:903,listed_count:18,favourites_count:5664,statuses_count:22919,created_at:mon feb 14 22:44:37 +0000 2011,utc_offset:3600,time_zone:amsterdam,geo_enabled:false,lang:en,contributors_enabled:false,is_translator:false,profile_background_color:c0deed,profile_background_image_url:http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png,profile_background_image_url_https:https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png,profile_background_tile:false,profile_link_color:0084b4,profile_sidebar_border_color:c0deed,profile_sidebar_fill_color:ddeef6,profile_text_color:333333,profile_use_background_image:true,profile_image_url:http:\/\/pbs.twimg.com\/profile_images\/2600097910\/image_normal.jpg,profile_image_url_https:https:\/\/pbs.twimg.com\/profile_images\/2600097910\/image_normal.jpg,default_profile:true,default_profile_image:false,following:null,follow_request_sent:null,notifications:null},geo:null,coordinates:null,place:null,contributors:null,is_quote_status:false,retweet_count:0,favorite_count:0,entities:{hashtags:[],urls:[],user_mentions:[],symbols:[]},favorited:false,retweeted:false,filter_level:low,lang:en,timestamp_ms:1452945648659}
Or is there any other way but using split? I would really appreciate your tips.
The error is as below.
16/09/18 22:49:37 ERROR TaskSetManager: Task 0 in stage 0.0 failed 1 times; aborting job
Hi hope I understand the question correctly, you are attempting to read a file and with the text mentioned above and then print the "text" mentioned in file containing json
If the above assumption is correct, here a simple code which would do this:
val matchingPattern = "(?i)(text:)(.+?)(,source:)".r
val tweets = scala.io.Source.fromPath("/home/tobby/data/shortTwitter.txt").getLines.reduceLeft(_+_)
matchingPattern.findAllIn(tweets).matchData foreach { m => println(m.group(2)) }
Hope it helps, if the above assumption is not correct please provide a sample input and expected output
I have a model called Order which has a DATETIME (Mysql) column called expiration
I do the following:
order = Order.last
wanted_time = Time.use_zone('Pacific Time (US & Canada)') { Time.now.end_of_day.to_datetime }
And wanted_time gives me: Thu, 20 Nov 2014 23:59:59 -0800
However, when I do:
order.expiration = wanted_time
When I try to fetch order.expiration, I get:
Fri, 21 Nov 2014 07:59:59 UTC +00:00
Why is that? If I specifically have defined the zone I wanted to store it with?
See this: How to change default timezone for Active Record in Rails?
Change application.rb to have the following lines... I'm guessing theres a difference between rails and db.
config.time_zone = 'Eastern Time (US & Canada)'
config.active_record.default_timezone = 'Eastern Time (US & Canada)'
I am trying to create a focus brush, similar to this http://bl.ocks.org/mbostock/1667367, using a json file.
I am having a problem to correctly read the date and create the xScale for the chart.
Say these are some of the dates in my Json file, extracted into an array.
//some dates
dates = ["Fri Jul 27 22:32:59 2012","Fri Jul 27 22:33:59 2012","Fri Jul 27 22:34:59 2012","Fri Jul 27 22:35:59 2012","Fri Jul 27 22:36:59 2012"];
//declare new date variable
var dates2 =[];
var format = d3.time.format("%a %b %d %H:%M:%S %Y");
dates.forEach(function(d){
dates2.push(format.parse(d));
});
console.log(d3.extent(dates_extents));
dates_extents = d3.extent(dates2);
x.domain(d3.extent(dates_extents);
x2.domain(x.domain);
Althought the dates_extents is printed in the console, I get this error:
TypeError: n.map is not a function [Break On This Error] ...t,n)},n.copy=function(){return ae(t)},n}function oe(t){return t.innerRadius}func... d3.v3.min.js (line 1)
Any ideas what might be the source of the problem and how I can solve it?
Thanks!