I trying to create a DB schema based on a JSON file - mysql
I am trying to create a DB based on a JSON file but I am messing up the keys and references. Can you guys have a look and tell me whats wrong with the create statement?
The json files are basically reviews or comments from people and there are 7 response objects and one cursor object in this json file.
JSON file:
{
"cursor": {
"prev": null,
"hasNext": false,
"next": "1542275432008325:1:0",
"hasPrev": false,
"total": null,
"id": "1542275432008325:1:0",
"more": false
},
"code": 0,
"response": [
{
"editableUntil": "2018-11-22T11:20:37",
"dislikes": 0,
"numReports": 0,
"likes": 1,
"message": "<p>A sinking pound, resignations by the bucketful railroading through a so-called agreement nobody wants. For the good of the country Mrs. May 'go and go now'.</p>",
"id": "4196155749",
"createdAt": "2018-11-15T11:20:37",
"author": {
"username": "mnaid-233509073ed3432027d48b1a83f5fbd2",
"about": "",
"name": "baggiebuoy",
"disable3rdPartyTrackers": true,
"isPowerContributor": false,
"joinedAt": "2017-05-03T10:40:52",
"profileUrl": "https://disqus.com/by/mnaid-233509073ed3432027d48b1a83f5fbd2/",
"url": "",
"location": "",
"isPrivate": false,
"signedUrl": "",
"isPrimary": true,
"isAnonymous": false,
"id": "250728493",
"avatar": {
"small": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-233509073ed3432027d48b1a83f5fbd2.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar32.png"
},
"isCustom": false,
"permalink": "https://disqus.com/api/users/avatars/mnaid-233509073ed3432027d48b1a83f5fbd2.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png",
"large": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-233509073ed3432027d48b1a83f5fbd2.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png"
}
}
},
"media": [],
"isSpam": false,
"isDeletedByAuthor": false,
"isDeleted": false,
"parent": null,
"isApproved": true,
"isFlagged": false,
"raw_message": "A sinking pound, resignations by the bucketful railroading through a so-called agreement nobody wants. For the good of the country Mrs. May 'go and go now'.",
"isHighlighted": false,
"canVote": false,
"thread": "7046384220",
"forum": "expressandstar",
"points": 1,
"moderationLabels": [],
"isEdited": true,
"sb": false
},
{
"editableUntil": "2018-11-22T10:37:59",
"dislikes": 0,
"numReports": 0,
"likes": 0,
"message": "<p>This could be heaven, or this could be hell.</p>",
"id": "4196048572",
"createdAt": "2018-11-15T10:37:59",
"author": {
"username": "mnaid-29e1c59be16c852670e3be302e8c303b",
"about": "",
"name": "Mordecai",
"disable3rdPartyTrackers": false,
"isPowerContributor": false,
"joinedAt": "2017-05-03T13:01:02",
"profileUrl": "https://disqus.com/by/mnaid-29e1c59be16c852670e3be302e8c303b/",
"url": "",
"location": "",
"isPrivate": false,
"signedUrl": "",
"isPrimary": true,
"isAnonymous": false,
"id": "250739494",
"avatar": {
"small": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-29e1c59be16c852670e3be302e8c303b.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar32.png"
},
"isCustom": false,
"permalink": "https://disqus.com/api/users/avatars/mnaid-29e1c59be16c852670e3be302e8c303b.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png",
"large": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-29e1c59be16c852670e3be302e8c303b.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png"
}
}
},
"media": [],
"isSpam": false,
"isDeletedByAuthor": false,
"isDeleted": false,
"parent": 4196013534,
"isApproved": true,
"isFlagged": false,
"raw_message": "This could be heaven, or this could be hell.",
"isHighlighted": false,
"canVote": false,
"thread": "7046384220",
"forum": "expressandstar",
"points": 0,
"moderationLabels": [],
"isEdited": false,
"sb": false
},
{
"editableUntil": "2018-11-22T10:36:50",
"dislikes": 0,
"numReports": 0,
"likes": 0,
"message": "<p>The crappest of crap deals.<br>Will never get through Parliament.</p><p>You are the weakest link May, goodbye.</p>",
"id": "4196044068",
"createdAt": "2018-11-15T10:36:50",
"author": {
"username": "mnaid-29e1c59be16c852670e3be302e8c303b",
"about": "",
"name": "Mordecai",
"disable3rdPartyTrackers": false,
"isPowerContributor": false,
"joinedAt": "2017-05-03T13:01:02",
"profileUrl": "https://disqus.com/by/mnaid-29e1c59be16c852670e3be302e8c303b/",
"url": "",
"location": "",
"isPrivate": false,
"signedUrl": "",
"isPrimary": true,
"isAnonymous": false,
"id": "250739494",
"avatar": {
"small": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-29e1c59be16c852670e3be302e8c303b.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar32.png"
},
"isCustom": false,
"permalink": "https://disqus.com/api/users/avatars/mnaid-29e1c59be16c852670e3be302e8c303b.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png",
"large": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-29e1c59be16c852670e3be302e8c303b.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png"
}
}
},
"media": [],
"isSpam": false,
"isDeletedByAuthor": false,
"isDeleted": false,
"parent": null,
"isApproved": true,
"isFlagged": false,
"raw_message": "The crappest of crap deals.\nWill never get through Parliament.\n\nYou are the weakest link May, goodbye.",
"isHighlighted": false,
"canVote": false,
"thread": "7046384220",
"forum": "expressandstar",
"points": 0,
"moderationLabels": [],
"isEdited": false,
"sb": false
},
{
"editableUntil": "2018-11-22T10:28:13",
"dislikes": 0,
"numReports": 0,
"likes": 0,
"message": "<p>We are all just prisoners here of our own device. 😁</p>",
"id": "4196013534",
"createdAt": "2018-11-15T10:28:13",
"author": {
"username": "mnaid-42ba513c42a0fd6558aa44b1de658140",
"about": "",
"name": "chaffwolf",
"disable3rdPartyTrackers": true,
"isPowerContributor": false,
"joinedAt": "2017-12-20T16:45:10",
"profileUrl": "https://disqus.com/by/mnaid-42ba513c42a0fd6558aa44b1de658140/",
"url": "",
"location": "",
"isPrivate": false,
"signedUrl": "",
"isPrimary": true,
"isAnonymous": false,
"id": "274657487",
"avatar": {
"small": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-42ba513c42a0fd6558aa44b1de658140.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar32.png"
},
"isCustom": false,
"permalink": "https://disqus.com/api/users/avatars/mnaid-42ba513c42a0fd6558aa44b1de658140.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png",
"large": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-42ba513c42a0fd6558aa44b1de658140.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png"
}
}
},
"media": [],
"isSpam": false,
"isDeletedByAuthor": false,
"isDeleted": false,
"parent": 4195981074,
"isApproved": true,
"isFlagged": false,
"raw_message": "We are all just prisoners here of our own device. 😁",
"isHighlighted": false,
"canVote": false,
"thread": "7046384220",
"forum": "expressandstar",
"points": 0,
"moderationLabels": [],
"isEdited": false,
"sb": false
},
{
"editableUntil": "2018-11-22T10:27:03",
"dislikes": 0,
"numReports": 0,
"likes": 0,
"message": "<p>Should be no deal end off.<br>Nobody could possibly be the winner so it's simple leave and that's it .<br>Their will always be wannabes who think they can do better in it for only their own powers not for the general country or the people of it .,</p>",
"id": "4196012501",
"createdAt": "2018-11-15T10:27:03",
"author": {
"username": "mnaid-90ecce8d5dad4396f681182cb470872c",
"about": "",
"name": "wanderer in eire",
"disable3rdPartyTrackers": true,
"isPowerContributor": false,
"joinedAt": "2017-05-12T20:20:43",
"profileUrl": "https://disqus.com/by/mnaid-90ecce8d5dad4396f681182cb470872c/",
"url": "",
"location": "",
"isPrivate": false,
"signedUrl": "",
"isPrimary": true,
"isAnonymous": false,
"id": "251694793",
"avatar": {
"small": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-90ecce8d5dad4396f681182cb470872c.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar32.png"
},
"isCustom": false,
"permalink": "https://disqus.com/api/users/avatars/mnaid-90ecce8d5dad4396f681182cb470872c.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png",
"large": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-90ecce8d5dad4396f681182cb470872c.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png"
}
}
},
"media": [],
"isSpam": false,
"isDeletedByAuthor": false,
"isDeleted": false,
"parent": null,
"isApproved": true,
"isFlagged": false,
"raw_message": "Should be no deal end off.\nNobody could possibly be the winner so it's simple leave and that's it .\nTheir will always be wannabes who think they can do better in it for only their own powers not for the general country or the people of it .,",
"isHighlighted": false,
"canVote": false,
"thread": "7046384220",
"forum": "expressandstar",
"points": 0,
"moderationLabels": [],
"isEdited": false,
"sb": false
},
{
"editableUntil": "2018-11-22T10:26:42",
"dislikes": 0,
"numReports": 0,
"likes": 0,
"message": "<p>Damned if she does, damned if she doesn't. All for a cause she does not believe in.<br>She will go down in history whatever happens. <br>The question is: who better to lead the country at such an important juncture??<br>Answers on a postcard!</p>",
"id": "4196012237",
"createdAt": "2018-11-15T10:26:42",
"author": {
"username": "mnaid-42ba513c42a0fd6558aa44b1de658140",
"about": "",
"name": "chaffwolf",
"disable3rdPartyTrackers": true,
"isPowerContributor": false,
"joinedAt": "2017-12-20T16:45:10",
"profileUrl": "https://disqus.com/by/mnaid-42ba513c42a0fd6558aa44b1de658140/",
"url": "",
"location": "",
"isPrivate": false,
"signedUrl": "",
"isPrimary": true,
"isAnonymous": false,
"id": "274657487",
"avatar": {
"small": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-42ba513c42a0fd6558aa44b1de658140.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar32.png"
},
"isCustom": false,
"permalink": "https://disqus.com/api/users/avatars/mnaid-42ba513c42a0fd6558aa44b1de658140.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png",
"large": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-42ba513c42a0fd6558aa44b1de658140.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png"
}
}
},
"media": [],
"isSpam": false,
"isDeletedByAuthor": false,
"isDeleted": false,
"parent": null,
"isApproved": true,
"isFlagged": false,
"raw_message": "Damned if she does, damned if she doesn't. All for a cause she does not believe in.\nShe will go down in history whatever happens. \nThe question is: who better to lead the country at such an important juncture??\nAnswers on a postcard!",
"isHighlighted": false,
"canVote": false,
"thread": "7046384220",
"forum": "expressandstar",
"points": 0,
"moderationLabels": [],
"isEdited": false,
"sb": false
},
{
"editableUntil": "2018-11-22T09:50:32",
"dislikes": 0,
"numReports": 0,
"likes": 0,
"message": "<p>Theresa May is using The Eagles Hotel California lyrics as the inspiration for her Brexit plan.</p><p>You can check out any time you like, but you can never leave.</p>",
"id": "4195981074",
"createdAt": "2018-11-15T09:50:32",
"author": {
"username": "mnaid-e82a88d937e60267fd2c866b01131ada",
"about": "",
"name": "Olly the cat",
"disable3rdPartyTrackers": false,
"isPowerContributor": false,
"joinedAt": "2017-05-03T10:27:23",
"profileUrl": "https://disqus.com/by/mnaid-e82a88d937e60267fd2c866b01131ada/",
"url": "",
"location": "",
"isPrivate": false,
"signedUrl": "",
"isPrimary": true,
"isAnonymous": false,
"id": "250727584",
"avatar": {
"small": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-e82a88d937e60267fd2c866b01131ada.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar32.png"
},
"isCustom": false,
"permalink": "https://disqus.com/api/users/avatars/mnaid-e82a88d937e60267fd2c866b01131ada.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png",
"large": {
"permalink": "https://disqus.com/api/users/avatars/mnaid-e82a88d937e60267fd2c866b01131ada.jpg",
"cache": "//a.disquscdn.com/1541535443/images/noavatar92.png"
}
}
},
"media": [],
"isSpam": false,
"isDeletedByAuthor": false,
"isDeleted": false,
"parent": null,
"isApproved": true,
"isFlagged": false,
"raw_message": "Theresa May is using The Eagles Hotel California lyrics as the inspiration for her Brexit plan.\n\nYou can check out any time you like, but you can never leave.",
"isHighlighted": false,
"canVote": false,
"thread": "7046384220",
"forum": "expressandstar",
"points": 0,
"moderationLabels": [],
"isEdited": false,
"sb": false
}
]
}
SQL Statement:
CREATE TABLE cursor(
code DOUBLE NOT NULL,
hasNext VARCHAR(50) NOT NULL,
next VARCHAR(50) NOT NULL,
hasPrev VARCHAR(50) NOT NULL,
id VARCHAR(50) NOT NULL,
more VARCHAR(50) NOT NULL,
PRIMARY KEY (hasNext),
FOREIGN KEY (code) REFERENCES dbjson(code)
)
CREATE TABLE small(
isCustom VARCHAR(50) NOT NULL,
permalink VARCHAR(100) NOT NULL,
cache VARCHAR(100) NOT NULL,
PRIMARY KEY (permalink),
FOREIGN KEY (username) REFERENCES avatar(isCustom)
)
CREATE TABLE moderationLabels(
editableUntil VARCHAR(50) NOT NULL,
FOREIGN KEY (code) REFERENCES response(editableUntil)
)
CREATE TABLE large(
isCustom VARCHAR(50) NOT NULL,
permalink VARCHAR(100) NOT NULL,
cache VARCHAR(100) NOT NULL,
PRIMARY KEY (permalink),
FOREIGN KEY (username) REFERENCES avatar(isCustom)
)
CREATE TABLE response(
code DOUBLE NOT NULL,
editableUntil VARCHAR(50) NOT NULL,
dislikes DOUBLE NOT NULL,
numReports DOUBLE NOT NULL,
likes DOUBLE NOT NULL,
message VARCHAR(200) NOT NULL,
id VARCHAR(50) NOT NULL,
createdAt VARCHAR(50) NOT NULL,
isSpam VARCHAR(50) NOT NULL,
isDeletedByAuthor VARCHAR(50) NOT NULL,
isDeleted VARCHAR(50) NOT NULL,
isApproved VARCHAR(50) NOT NULL,
isFlagged VARCHAR(50) NOT NULL,
raw_message VARCHAR(200) NOT NULL,
isHighlighted VARCHAR(50) NOT NULL,
canVote VARCHAR(50) NOT NULL,
thread VARCHAR(50) NOT NULL,
forum VARCHAR(50) NOT NULL,
points DOUBLE NOT NULL,
isEdited VARCHAR(50) NOT NULL,
sb VARCHAR(50) NOT NULL,
PRIMARY KEY (editableUntil),
FOREIGN KEY (code) REFERENCES dbjson(code)
)
CREATE TABLE author(
editableUntil VARCHAR(50) NOT NULL,
username VARCHAR(50) NOT NULL,
about VARCHAR(50) NOT NULL,
name VARCHAR(50) NOT NULL,
disable3rdPartyTrackers VARCHAR(50) NOT NULL,
isPowerContributor VARCHAR(50) NOT NULL,
joinedAt VARCHAR(50) NOT NULL,
profileUrl VARCHAR(100) NOT NULL,
url VARCHAR(50) NOT NULL,
location VARCHAR(50) NOT NULL,
isPrivate VARCHAR(50) NOT NULL,
signedUrl VARCHAR(50) NOT NULL,
isPrimary VARCHAR(50) NOT NULL,
isAnonymous VARCHAR(50) NOT NULL,
id DOUBLE NOT NULL,
PRIMARY KEY (username),
FOREIGN KEY (code) REFERENCES response(editableUntil)
)
CREATE TABLE avatar(
username VARCHAR(50) NOT NULL,
isCustom VARCHAR(50) NOT NULL,
permalink VARCHAR(100) NOT NULL,
cache VARCHAR(100) NOT NULL,
PRIMARY KEY (isCustom),
FOREIGN KEY (editableUntil) REFERENCES author(username)
)
CREATE TABLE media(
editableUntil VARCHAR(50) NOT NULL,
FOREIGN KEY (code) REFERENCES response(editableUntil)
)
CREATE TABLE dbjson(
code DOUBLE NOT NULL,
PRIMARY KEY (code))
When you define foreign keys, you must define the parent table before you can define a child table that references it. Your tables are defined in an order such that several tables references their parent before the parent is created.
Next problem: cursor is a reserved keyword in MySQL. You must delimit it with back-ticks like this:
CREATE TABLE `cursor`(
...
Next problem: Several tables declare a foreign key on a column that doesn't exist in the table. For example, author has:
FOREIGN KEY (code) REFERENCES response(editableUntil)
But the author table has no column called code. The column must exist in the table before you can declare a foreign key on it. The same problem occurs in tables avatar, small, moderationLabels, large, and media.
Related
Retrieving the "Link_ID" within an API
I am using the Reddit API called Pushshift (https://github.com/pushshift/api) I am trying to get the "LINK_ID" for Reddit post based on the title of the search. For example, I want to find out all Reddit posts that contain the word "Trump" in the title - and I want to get the LINK_ID for each one of these posts. When I search : https://api.pushshift.io/reddit/search/submission/?title=trump I get the following results: "data": [ { "all_awardings": [], "allow_live_comments": false, "author": "rajacreator", "author_flair_css_class": null, "author_flair_richtext": [], "author_flair_text": null, "author_flair_type": "text", "author_fullname": "t2_ouuxidx4", "author_is_blocked": false, "author_patreon_flair": false, "author_premium": false, "awarders": [], "can_mod_post": false, "contest_mode": false, "created_utc": 1663481539, "domain": "rajacreator.com", "full_link": "https://www.reddit.com/r/news/comments/xh8ui0/servants_of_the_damned_overview_trump_and_the/", "gildings": {}, "id": "xh8ui0", "is_created_from_ads_ui": false, "is_crosspostable": false, "is_meta": false, "is_original_content": false, "is_reddit_media_domain": false, "is_robot_indexable": false, "is_self": false, "is_video": false, "link_flair_background_color": "", "link_flair_richtext": [], "link_flair_text_color": "dark", "link_flair_type": "text", "locked": false, "media_only": false, "no_follow": true, "num_comments": 0, "num_crossposts": 0, "over_18": false, "parent_whitelist_status": "all_ads", "permalink": "/r/news/comments/xh8ui0/servants_of_the_damned_overview_trump_and_the/", "pinned": false, "pwls": 6, "removed_by_category": "automod_filtered", "retrieved_on": 1663481550, "score": 1, "selftext": "", "send_replies": false, "spoiler": false, "stickied": false, "subreddit": "news", "subreddit_id": "t5_2qh3l", "subreddit_subscribers": 25211187, "subreddit_type": "public", "thumbnail": "default", "title": "Servants of the Damned overview: Trump and the giant law firm he actually paid", "total_awards_received": 0, "treatment_tags": [], "upvote_ratio": 1.0, "url": "https://rajacreator.com/servants-of-the-damned-review-trump-and-the-giant-law-firm-he-actually-paid/?utm_source=SocialAutoPoster", "url_overridden_by_dest": "https://rajacreator.com/servants-of-the-damned-review-trump-and-the-giant-law-firm-he-actually-paid/?utm_source=SocialAutoPoster", "whitelist_status": "all_ads", "wls": 6 }, There seems to be nothing here about "LINK_ID". Can someone please show me how to get the LINK_ID? Thank you! Note: In the future - once I find the LINK_ID for a specific Reddit link, I would then like to extract all the comments posted for that Reddit link (e.g. https://www.reddit.com/r/pushshift/comments/bcwqxb/get_all_comments_from_submission/)
Nested Json data to postgresql using python
How can i import my json file to a postgresql using python my data will look likes { "blocked_by": false, "blocking": false, "contributors_enabled": false, "created_at": "Thu Dec 17 06:32:35 +0000 2020", "default_profile": true, "default_profile_image": false, "description": "seo", "entities": { "description": { "urls": [] } }, "favourites_count": 12, "follow_request_sent": false, "followers_count": 56, "following": false, "friends_count": 1344, "geo_enabled": false, "has_extended_profile": true, "id": 1339458394508374018, "id_str": "1339458394508374018", "is_translation_enabled": false, "is_translator": false, "lang": null, "listed_count": 3, "live_following": false, "location": "", "muting": false, "name": "katheryn myle", "notifications": false, "profile_background_color": "F5F8FA", "profile_background_image_url": null, "profile_background_image_url_https": null, "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/1343124937389842432/30cfUmGe_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1343124937389842432/30cfUmGe_normal.jpg", "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "protected": false, "screen_name": "KatherynMyle", "status": { "contributors": null, "coordinates": null, "created_at": "Tue Apr 05 10:17:04 +0000 2022", "entities": { "hashtags": [], "symbols": [], "urls": [ { "display_url": "twitter.com/i/web/status/1\u2026", "expanded_url": "https://twitter.com/i/web/status/1511286791319564293", "indices": [ 117, 140 ], "url": "shortened url" } ], "user_mentions": [ { "id": 2247373052, "id_str": "2247373052", "indices": [ 0, 12 ], "name": "GDevelop", "screen_name": "GDevelopApp" } ] }, "favorite_count": 0, "favorited": false, "geo": null, "id": 1511286791319564293, "id_str": "1511286791319564293", "in_reply_to_screen_name": "GDevelopApp", "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": 2247373052, "in_reply_to_user_id_str": "2247373052", "is_quote_status": false, "lang": "en", "place": null, "retweet_count": 0, "retweeted": false, "source": "Twitter Web App", "text": "#GDevelopApp Hello\nHow are you fine I am a link builder and we are selling links and posts on high-quality sites if\u2026 ", "truncated": true }, "statuses_count": 382, "time_zone": null, "translator_type": "none", "url": null, "utc_offset": null, "verified": false, "withheld_in_countries": [] I want to create a table with it have ing the index or key name as the column name and also i want only some index data from it not all (like utc _offset,place,etc.).so i want to insert the specific data i wanted to postgresql https://drive.google.com/file/d/1OkJFfHyU4Eb-V3mrU2qJ_jwBaQQkunJa/view?usp=drivesdk this is the json file
Your question is very vague and I can't come up with a short answer except this one: pSQL does support JSON as a data type: https://www.postgresql.org/docs/current/datatype-json.html https://www.postgresql.org/docs/current/functions-json.html You could just store the whole json as a single json element and select/convert/whatever within postgre later on. How to insert JSONB into Postgresql with Python?
Azure Congnitive Service Skill not persisting custom skill value
I have create a custom skill for my Azure Cognitive Service. After creating the datasouce, index, indexer and running the indexer I can see that my function is being called and it is outputting the correct information, but when I search the index the field that is connected to the custom skill is empty. This is my indexer definition: { "name": "idxdocs", "description": null, "dataSourceName": "dsdocs", "skillsetName": "skillset-procuradores", "targetIndexName": "customer-documents", "disabled": null, "schedule": null, "parameters": { "batchSize": null, "maxFailedItems": -1, "maxFailedItemsPerBatch": null, "base64EncodeKeys": null, "configuration": { "indexedFileNameExtensions": ".pdf,.docx,.doc", "dataToExtract": "contentAndMetadata", "failOnUnprocessableDocument": false, "failOnUnsupportedContentType": false, "indexStorageMetadataOnlyForOversizedDocuments": true, "allowSkillsetToReadFileData": true } }, "fieldMappings": [], "outputFieldMappings": [ { "sourceFieldName": "/document/content/procuradores", "targetFieldName": "procuradores" } ], "cache": null, "encryptionKey": null } This is my index: { "name": "customer-documents", "fields": [ { "name": "key", "type": "Edm.String", "facetable": true, "filterable": true, "key": true, "retrievable": true, "searchable": false, "sortable": true, "analyzer": null, "indexAnalyzer": null, "searchAnalyzer": null, "synonymMaps": [], "fields": [] }, { "name": "content", "type": "Edm.String", "facetable": false, "filterable": false, "key": false, "retrievable": true, "searchable": true, "sortable": false, "analyzer": "es.lucene", "indexAnalyzer": null, "searchAnalyzer": null, "synonymMaps": [], "fields": [] }, { "name": "procuradores", "type": "Edm.String", "facetable": false, "filterable": true, "key": false, "retrievable": true, "searchable": false, "sortable": false, "analyzer": null, "indexAnalyzer": null, "searchAnalyzer": null, "synonymMaps": [], "fields": [] } ], "suggesters": [], "scoringProfiles": [], "defaultScoringProfile": null, "corsOptions": null, "analyzers": [], "charFilters": [], "tokenFilters": [], "tokenizers": [], "similarity": { "#odata.type": "#Microsoft.Azure.Search.BM25Similarity", "k1": null, "b": null }, "encryptionKey": null, "#odata.etag": "\"0x8D98BC85E9F6996\"" } My skill set definition: { "name": "skillset-procuradores", "description": "", "skills": [ { "#odata.type": "#Microsoft.Skills.Custom.WebApiSkill", "name": "procuradores", "description": "", "context": "/document", "uri": "my url ommited for secure reasons", "httpMethod": "POST", "timeout": "PT3M50S", "batchSize": 1, "degreeOfParallelism": null, "inputs": [ { "name": "text", "source": "/document/content" } ], "outputs": [ { "name": "procuradores", "targetName": "procuradores" } ], "httpHeaders": {} } ], "cognitiveServices": null, "knowledgeStore": null, "encryptionKey": null } And finally my function output: { "values": [ { "recordId": "1", "data": { "procuradores": "people's names" } } ] } What am I missing?
Rafael, can you give debug sessions a try? I suspect there is something happening in your custom skill that means the data is not coming back. By using this, you can test the actual input and output. https://learn.microsoft.com/en-us/azure/search/cognitive-search-debug-session
This is just a hunch, but in the indexer definition, try changing your output field mapping sourceFieldName from "/document/content/procuradores" to "/document/procuradores". The context that you are giving in the skill is just "/document" so it should append the output onto that.
Network IP addresses with jq bash command from docker inspect
I want to process docker inspect <container1> <container2> out to retrieve via jq command exposed ports and IPAddresses assigned to defined networks. So far I've got jq '{(.[].Name): getpath(path(.[].NetworkSettings.Networks)) | .[].IPAddress}' which is clearly far from enough. p.s. I know its easy to write a script that would give such output but I'm curious how to do that using jq Expected output { # .[].Name "/prometheus_prometheus_1": [ [ # .[].Config.ExposedPorts "9090/tcp" ], { # key = .[].NetworkSettings.Networks.<index> # value = .[].NetworkSettings.Networks[].IPAddress "prometheus_internal": "172.21.0.4", "system-information": "172.28.0.2" } ] } Input: [ { "Id": "c2f86bce68441f50f973ae89a835d052e74c083e35b5438f3cec2497bcac79fa", "Created": "2018-06-12T05:33:19.811448302Z", "Path": "/bin/prometheus", "Args": [ "--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--web.console.libraries=/usr/share/prometheus/console_libraries", "--web.console.templates=/usr/share/prometheus/consoles" ], "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 13217, "ExitCode": 0, "Error": "", "StartedAt": "2018-06-12T05:33:20.264845701Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:44a4c1c270dcb5e8caf51e3205fa23343cc903f3823eab53ae204f304d16f62a", "ResolvConfPath": "/var/lib/docker/containers/c2f86bce68441f50f973ae89a835d052e74c083e35b5438f3cec2497bcac79fa/resolv.conf", "HostnamePath": "/var/lib/docker/containers/c2f86bce68441f50f973ae89a835d052e74c083e35b5438f3cec2497bcac79fa/hostname", "HostsPath": "/var/lib/docker/containers/c2f86bce68441f50f973ae89a835d052e74c083e35b5438f3cec2497bcac79fa/hosts", "LogPath": "/var/lib/docker/containers/c2f86bce68441f50f973ae89a835d052e74c083e35b5438f3cec2497bcac79fa/c2f86bce68441f50f973ae89a835d052e74c083e35b5438f3cec2497bcac79fa-json.log", "Name": "/prometheus_prometheus_1", "RestartCount": 0, "Driver": "overlay2", "Platform": "linux", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [ "prometheus_prometheus-data:/prometheus:rw", "/home/sebastian/projects/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:rw" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": {} }, "NetworkMode": "prometheus_internal", "PortBindings": {}, "RestartPolicy": { "Name": "always", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": [], "CapAdd": null, "CapDrop": null, "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": null, "GroupAdd": null, "IpcMode": "shareable", "Cgroup": "", "Links": null, "OomScoreAdj": 0, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": null, "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 0, "Memory": 0, "NanoCpus": 0, "CgroupParent": "", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": null, "DeviceCgroupRules": null, "DiskQuota": 0, "KernelMemory": 0, "MemoryReservation": 0, "MemorySwap": 0, "MemorySwappiness": null, "OomKillDisable": false, "PidsLimit": 0, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0 }, "GraphDriver": { "Data": { "LowerDir": "/var/lib/docker/overlay2/73c16b0735eb15494741e5b821a9df3be8cebb716219ac206d9ee0042e4166ac-init/diff:/var/lib/docker/overlay2/03c66042305488fea2fe693246e90374743fbb7836ab31e4ee5cdbfbb1c97785/diff:/var/lib/docker/overlay2/80821a00b79334493729e671dfb66cc878aec19fe2d0dfbaf4cde187522b06e8/diff:/var/lib/docker/overlay2/f5071cce5513fb3c465ad883297157271ef48c6f59cd88768068224d850c6903/diff:/var/lib/docker/overlay2/c9a0cc617abb45fbd4572abb41c7281c96bc9d59c109270c7f1ec7a99f5b055b/diff:/var/lib/docker/overlay2/5b425f8a440139a8fdcdd7f7fee8189565e9e00304e96f4cfee86ff6b861e068/diff:/var/lib/docker/overlay2/e85740d997591d2f60bfa61ea0c82a58636bccaaba970e55f12cf0ff24565c70/diff:/var/lib/docker/overlay2/e9c362d971c460b02111122b241888392d77c5b8545ed520c3d74660b16db9b2/diff:/var/lib/docker/overlay2/5001184fc8ddd25204626f419c1530b970cb37f936a196fb1173f2774310085e/diff:/var/lib/docker/overlay2/ed2e676f09a4583f2194e752083a7c95be80131f055891dc39f9fed5560f5a73/diff", "MergedDir": "/var/lib/docker/overlay2/73c16b0735eb15494741e5b821a9df3be8cebb716219ac206d9ee0042e4166ac/merged", "UpperDir": "/var/lib/docker/overlay2/73c16b0735eb15494741e5b821a9df3be8cebb716219ac206d9ee0042e4166ac/diff", "WorkDir": "/var/lib/docker/overlay2/73c16b0735eb15494741e5b821a9df3be8cebb716219ac206d9ee0042e4166ac/work" }, "Name": "overlay2" }, "Mounts": [ { "Type": "volume", "Name": "prometheus_prometheus-data", "Source": "/var/lib/docker/volumes/prometheus_prometheus-data/_data", "Destination": "/prometheus", "Driver": "local", "Mode": "rw", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/home/sebastian/projects/prometheus/prometheus.yml", "Destination": "/etc/prometheus/prometheus.yml", "Mode": "rw", "RW": true, "Propagation": "rprivate" } ], "Config": { "Hostname": "c2f86bce6844", "Domainname": "", "User": "nobody", "AttachStdin": false, "AttachStdout": false, "AttachStderr": false, "ExposedPorts": { "9090/tcp": {} }, "Tty": false, "OpenStdin": false, "StdinOnce": false, "Env": [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ], "Cmd": [ "--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--web.console.libraries=/usr/share/prometheus/console_libraries", "--web.console.templates=/usr/share/prometheus/consoles" ], "Image": "prom/prometheus:v2.3.0", "Volumes": { "/etc/prometheus/prometheus.yml": {}, "/prometheus": {} }, "WorkingDir": "/prometheus", "Entrypoint": [ "/bin/prometheus" ], "OnBuild": null, "Labels": { "com.docker.compose.config-hash": "b8ff39fd239739ae66051752c1b49b0119862b8f21678fb42400c907d25b1c57", "com.docker.compose.container-number": "1", "com.docker.compose.oneoff": "False", "com.docker.compose.project": "prometheus", "com.docker.compose.service": "prometheus", "com.docker.compose.version": "1.16.1", "maintainer": "The Prometheus Authors <prometheus-developers#googlegroups.com>" } }, "NetworkSettings": { "Bridge": "", "SandboxID": "7be77954d0e6990f565a8526b157236140238cabf0954629598ce3bb9aaae885", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": { "9090/tcp": null }, "SandboxKey": "/var/run/docker/netns/7be77954d0e6", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "", "Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "", "Networks": { "prometheus_internal": { "IPAMConfig": null, "Links": null, "Aliases": [ "prometheus", "c2f86bce6844" ], "NetworkID": "145c59986c16003f9a3a30e6b671c34b56225d90243fba88fc295c49b4dcfff6", "EndpointID": "8e3ac8bd4719ea85358c20a8141c37377dd327265916d41ee5d1db6b7496898c", "Gateway": "172.21.0.1", "IPAddress": "172.21.0.4", "IPPrefixLen": 16, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "MacAddress": "02:42:ac:15:00:04", "DriverOpts": null }, "system-information": { "IPAMConfig": null, "Links": null, "Aliases": [ "prometheus", "c2f86bce6844" ], "NetworkID": "f6e67721cb73da7e8d4eacdb709de240014af9af35e66216e6186300c72de330", "EndpointID": "2c31369f8175f2b38a5bc966a8306c8f0f11ac4c264e6ce735d00c87b8a6f44f", "Gateway": "172.28.0.1", "IPAddress": "172.28.0.2", "IPPrefixLen": 16, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "MacAddress": "02:42:ac:1c:00:02", "DriverOpts": null } } } } ]
jq solution: jq '.[] | {(.Name) : [(.Config.ExposedPorts | keys), (.NetworkSettings.Networks | to_entries | map({(.key): .value.IPAddress}) | add) ] }' input The output: { "/prometheus_prometheus_1": [ [ "9090/tcp" ], { "prometheus_internal": "172.21.0.4", "system-information": "172.28.0.2" } ] }
How to create schema for this Json data in Hive?
{ "filter_level": "low", "retweeted": false, "in_reply_to_screen_name": "paperboatdrinks", "possibly_sensitive": false, "truncated": false, "lang": "und", "in_reply_to_status_id_str": "595491441440227328", "id": 595491657585299500, "in_reply_to_user_id_str": "1484637054", "timestamp_ms": "1430811249525", "in_reply_to_status_id": 595491441440227300, "created_at": "Tue May 05 07:34:09 +0000 2015", "favorite_count": 0, "place": null, "coordinates": null, "text": "#paperboatdrinks ������������������������������������", "contributors": null, "geo": null, "entities": { "trends": [], "symbols": [], "urls": [], "hashtags": [], "user_mentions": [ { "id": 1484637054, "name": "Paper Boat", "indices": [ 0, 16 ], "screen_name": "paperboatdrinks", "id_str": "1484637054" } ] }, "source": "Twitter for Android", "favorited": false, "in_reply_to_user_id": 1484637054, "retweet_count": 0, "id_str": "595491657585299456", "user": { "location": "", "default_profile": true, "statuses_count": 28, "profile_background_tile": false, "lang": "en", "profile_link_color": "0084B4", "profile_banner_url": "https://pbs.twimg.com/profile_banners/3099469722/1429881891", "id": 3099469722, "following": null, "favourites_count": 2, "protected": false, "profile_text_color": "333333", "verified": false, "description": "3d visualizer", "contributors_enabled": false, "profile_sidebar_border_color": "C0DEED", "name": "VICKY GUSAIN", "profile_background_color": "C0DEED", "created_at": "Fri Mar 20 11:43:14 +0000 2015", "default_profile_image": false, "followers_count": 3, "profile_image_url_https": "https://pbs.twimg.com/profile_images/591593652138344448/_JhM1YhS_normal.jpg", "geo_enabled": true, "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "follow_request_sent": null, "url": null, "utc_offset": null, "time_zone": null, "notifications": null, "profile_use_background_image": true, "friends_count": 9, "profile_sidebar_fill_color": "DDEEF6", "screen_name": "VickyHanumant", "id_str": "3099469722", "profile_image_url": "http://pbs.twimg.com/profile_images/591593652138344448/_JhM1YhS_normal.jpg", "listed_count": 0, "is_translator": false } }` But I had created schema but not accept it properly. create external table if not exists paper5 ( filter_level string, retweeted boolean, in_reply_to_screen_name string, possibly_sensitive boolean, truncated boolean, lang string, in_reply_to_status_id_str string, id int, in_reply_to_user_id_str string, timestamp_ms string, in_reply_to_status_id int, created_at string, favorite_count int, place string, coordinates string, text string, contributors string, geo string, entities struct, symbols: array, urls: array, hashtags: array, user_mentions: array>, screen_name: string, id_str: string>>, sources string, favorited boolean, in_reply_to_user_id int, retweet_count int, id_str string, user struct ) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.JsonSerde' location '/paper5/'; It showing error at entities and user_mentions field Please let me know to create a table in hive.
This link could help you it explains how you can insert data (json file) into hbase using hive QL with json_object