How to query Couchbase using Java API - couchbase

I'm trying to query from the Couchbase example beer-sample.
This query works fine in the Couchbase browser UI:
select category, style from `beer-sample` where style like 'Imperial%'
Results:
[
{
"category": "North American Ale",
"style": "Imperial or Double India Pale Ale"
},
...
]
But when I transplant the query into java, I get very peculiar results. (Yes, I know I'm opening/closing connection in the wrong place, just doing this for quick exploration of Couchbase syntax/features).
Java code:
#RequestMapping("/hellocouchbase")
public ResponseEntity<List<JsonObject>> metrics() {
Cluster cluster = CouchbaseCluster.create();
cluster.authenticate(username, passwd);
Bucket bucket = cluster.openBucket("beer-sample");
N1qlQueryResult result = bucket.query(N1qlQuery.simple("select category, style from `beer-sample` where style like 'Imperial%'"));
List<N1qlQueryRow> results = result.allRows();
List<JsonObject> answer = new ArrayList<>(results.size());
for(N1qlQueryRow row:results) {
answer.add(row.value());
}
cluster.disconnect();
return ResponseEntity.status(200).body(answer);
}
Results:
[
{"cryptoManager":null,"empty":false,"names":["style","category"]},{"cryptoManager":null,"empty":false,"names":["style","category"]},
...
]
Can someone explain how to make the java query produce the same results as the direct query?

Try to create a new user and add all privileges to it (just to be sure that you are not hitting any security restriction).
Your code works for me:
Cluster cluster = CouchbaseCluster.create();
cluster.authenticate("test", "couchbase"); //user and password that I created
Bucket bucket = cluster.openBucket("beer-sample");
N1qlQueryResult result = bucket.query(N1qlQuery.simple("select category, style from `beer-sample` where style like 'Imperial%'"));
List<N1qlQueryRow> results = result.allRows();
List<JsonObject> answer = new ArrayList<>(results.size());
for(N1qlQueryRow row:results) {
answer.add(row.value());
System.out.println(row);
}
cluster.disconnect();
Output:
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
...
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double Red Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}
{"category":"North American Ale","style":"Imperial or Double India Pale Ale"}

For some reason, changing this
answer.add(row.value());
to this
answer.add(row.value().toMap());
fixed it for me. No idea why, given that the original version apparently works fine for other people here.
Complete solution for reference:
#RequestMapping("/hellocouchbase")
public ResponseEntity<List<Map<String,Object>>> metrics() {
Cluster cluster = CouchbaseCluster.create();
cluster.authenticate(username, passwd);
Bucket bucket = cluster.openBucket("beer-sample");
N1qlQueryResult result = bucket.query(N1qlQuery.simple("select category, style from `beer-sample` where style like 'Imperial%'"));
List<N1qlQueryRow> results = result.allRows();
List<Map<String,Object>> answer = new ArrayList<>(results.size());
for(N1qlQueryRow row:results) {
answer.add(row.value().toMap());
}
cluster.disconnect();
return ResponseEntity.status(200).body(answer);
}

Related

Error on heroku when trying to npm run migrate to deploy my app

I'm trying to deploy my app on Heroku and I'm getting a type error when I npm run migrate on heroku run bash.
The error on the terminal is saying that it's a SQL typo error but I cannot see what is the error. I've tried to change the quotes, but it's not working either. Any help will be welcome.
error:
require("dotenv").config();
const mysql = require("mysql");
const DB_HOST = process.env.DB_HOST;
const DB_USER = process.env.DB_USER;
const DB_PASS = process.env.DB_PASS;
const DB_NAME = process.env.DB_NAME;
const con = mysql.createConnection({
host: DB_HOST || "127.0.0.1",
user: DB_USER || "root",
password: DB_PASS || "root",
database: DB_NAME || "5aday",
multipleStatements: true
});
con.connect(function(err) {
if (err) throw err;
console.log("Connected!");
let sql = `DROP TABLE if exists seasons; CREATE TABLE into seasons (id INT NOT NULL AUTO_INCREMENT, image VARCHAR(255) NOT NULL, title VARCHAR(255) NOT NULL, description VARCHAR(1000) NOT NULL, list VARCHAR(1000)NOT NULL, PRIMARY KEY (id));`;
`INSERT INTO seasons (image, title,https description, list) VALUES ('https://www.juiceplus.com/content/juiceplusblog/en/2019/3/the-ultimate-fruit-and-vegetable-season-calendar/_jcr_content/par/image.img.jpg/1553178913562.jpg', 'FRUITS AND VEGETABLES AVAILABLE IN WINTER', 'Fruits and vegetables available in the winter months consist of lots of fruits you’d typically associate with Christmas (clementines, cranberries), but also lots of earthy vegetables like turnips, parsnips and leeks. There are some lovely winter warming recipes you can cook with these!', 'Apple
Pears
Clementines
Cranberries
Dates
Beetroot
Brussels sprouts
Celeriac
Celery
Kale
Leek
Parsnip
Turnip
Sweet potato
Grapefruit
Lemon
Orange
Rhubarb
Kiwi
Passion Fruit
Pomegranate
Beetroot
Brussels sprouts
Celeriac
Celery
Kale
Leek
Parsnip
Turnip
Sweet potato
Cauliflower
Chicory
Jerusalem artichoke');`
`INSERT INTO seasons (image, title, description, list) VALUES ('https://www.juiceplus.com/content/juiceplusblog/en/2019/3/the-ultimate-fruit-and-vegetable-season-calendar/_jcr_content/par/image_1056369294.img.jpg/1553178984905.jpg', 'FRUITS AND VEGETABLES AVAILABLE IN SPRING', 'Coming into to spring and lots of fresh spring greens come into season, like watercress, spinach and lettuce. New potatoes also come in the spring, which make for a delicious buttery dish or potato salad with watercress.', 'Grapefruit
Lemon
Orange
Rhubarb
Passion Fruit
Pomegranate
Apricot
Nectarine
Brussels sprouts
Cauliflower
Chicory
Jerusalem artichoke
Spinach
Watercress
Purple sprouting broccoli
New potatoes
Potatoes
Carrot
Lambs lettuce
Lettuce');`
`INSERT INTO seasons (image, title, description, list) VALUES ('https://www.juiceplus.com/content/juiceplusblog/en/2019/3/the-ultimate-fruit-and-vegetable-season-calendar/_jcr_content/par/image_1788290349.img.jpg/1553179052619.jpg', 'FRUITS AND VEGETABLES AVAILABLE IN SUMMER', 'As summer comes around, you are certainly not limited in options when it comes to seasonal fruit and vegetables. This is prime time for strawberries, nectarines and apricots – as well as asparagus, peas and runner beans.', 'Apricot
Blackcurrants
Gooseberry
Nectarine
Rhubarb
Strawberry
Tomato
Cherries
Raspberry
Redcurrant
Watermelon
Elderberries
Figs
Peach
Asparagus
Aubergine
Carrot
Courgettes
Lambs lettuce
Lettuce
New potatoes
Peas
Potatoes
Runner beans
Beetroot
Cavolo Nero
Fennel
Celery');`
`INSERT INTO seasons (image, title, description, list) VALUES ('https://www.juiceplus.com/content/juiceplusblog/en/2019/3/the-ultimate-fruit-and-vegetable-season-calendar/_jcr_content/par/image_1229398484.img.jpg/1553179114378.jpg', 'FRUITS AND VEGETABLES AVAILABLE IN AUTUMN', 'As we turn into autumn, the choice is still plentiful as many vegetables are still giving following the summer months. Sweetcorn is exclusively at it’s best in September and apple and pears come back into full season.', 'Apple
Blackberry
Elderberries
Figs
Gooseberry
Nectarine
Peach
Redcurrant
Plum
Raspberry
Strawberry
Tomato
Pear
Plum
Aubergine
Beetroot
Cavolo Nero
Celeriac
Celery
Courgettes
Fennel
Artichoke
Lambs lettuce
Leek
Peas
Lettuce
Runner beans
Sweetcorn
Brussels sprouts
Kale
Parsnip
Pumpkin
Sweet potato ');`
con.query(sql, function (err, result) {
if (err) throw err;
console.log("Table creation `seasons` was successful!");
console.log("Closing...");
});
con.end();
});
You dont assigne your sql statement to any variable. The first statement is assigned to the sql variable but not the following. Your are terminating the statement after the first create table.
There is an "INTO" too much in the first CREATE TABLE ("CREATE TABLE into seasons") statement if i am seeing this right.

'utf-8' codec can't decode byte 0xca in position 972: invalid continuation byte: using os, shutil, dictionary to move files

i have a question and i dont know if i can execute it on python
so i have files in the form of cik number-year.txt
and i have created directories of filenames matching firm names
and i have a spreadsheet of filename matching cik
I wrote a piece of code that should perform what I described, but i ran into an error which states UnicodeDecodeError: 'utf-8' codec can't decode byte 0xca in position 972: invalid continuation byte
Here is the csv file that im trying to read
company name ,cik,cik 2 ,missing ,notes
Deere,315189,,,
H.J. Heinz,46640,1637459,,
Bestfoods,25350,,,
Bayer Corporation,,,,
Sealed Air,1012100,,missing 1994-1997,
Eli Lilly,59478,,,
Campbell Soup,16732,,"missing 96, 97, 98, 99,00, 01",
Honeywell,48305,773840,overlapping years as have honeywell central and honeywell international,
Kellogg,55067,,,
Union Carbide,100790,,"missing 10-ks from 1993-1996, 2001 ",
Cooper Industries,24454,1141982,"missing 1993-2002, 2013-2018",
North American Philips,,,,
Intel,50863,,,
Amerada Hess,4447,,"missing 95, 97, 00",
Martin Marietta,916076,,1994-2002 ,
PPG Industries,79879,,"1997, 2000, 2001",
Litton Industries,59880,,"1995, 2001-2018",
Reynolds Metals,83604,,"95, 96, 97, 98, 00-18",
Warner-Lambert,104669,,"96, 97, 01-18",
Quaker Oats,81371,,,
Levi Strauss,94845,,before 2000,
Northrop Grumman,72945,,2002-2018,
Stone Container,94610,,,
LTV,,,,
American Cyanamid,4829,,,AMERICAN CYANAMID CO merged with American Home Products (AHP) in 1994.
Gillette,41499,,"96-98, 06-18","On October 1, 2005, Procter & Gamble finalized its merger with the Gillette Company."
Johnson Controls,53669,,"97, 98,99, 00, 01, 17-18 ",
Coca-Cola Enterprises,1491675,,"before 2011, after 2016 ",
BASF,,,,
Dana,26780,,95,
Champion International,19150,,nearly no data apart from 95,
Scott Paper,87949,,,The company was acquired by the Kimberly-Clark Corporation in 1995
Lyondell Chemical,842635,1489393,"95-98,09-18","LyondellBasell was formed in December 2007 by the acquisition of Lyondell Chemical Company by Basell Polyolefins for $12.7 billion.[7] As of 2016, Lyondell was the third largest independent chemical manufacturer in the United States.[8]"
Black & Decker,93556,12355,"00,01 ",2010 – Black & Decker merges with Stanley Works to become Stanley Black & Decker
Fort James,53117,,,"In 1997, the Fort Howard Paper Company and the James River Corporation merged to form the Fort James Corporation.[1][4] Fort Howard was headquartered in Green Bay and James River in Richmond,In 2000, the Fort James Corporation was acquired by Georgia-Pacific for $11 billion;[1][4] GP is based in Atlanta, Georgia. Virginia."
Mead,64394,,"missing 2001+, strange. ","missing 2005+, strange. "
Chiquita Brands Intl.,101063,,"98,01,06-18",
Dresser Industries,30099,,,"In 1998, Dresser merged with its main rival Halliburton,[1] Halliburton sold many of former Dresser non ""oil patch"" divisions, retaining the M W Kellogg Engineering and Construction Company and the Dresser oil-patch products and services that complemented Halliburton's energy and natural resource businesses. In 2001 Halliburton sold five separate, but somewhat related former Dresser non ""oil patch"" divisions, to an investment banking firm. Those five operations later took the name ""Dresser Inc."" In October 2010, Dresser Inc., was acquired by General Electric.[2] It is headquartered in Addison, Texas.[3]"
R.R. Donnelley & Sons,29669,,"95,97-02",
Tyson Foods,100493,,"95-99,01",
Compaq Computer,714154,,"96, 97","Struggling to keep up in the price wars against Dell, as well as with a risky acquisition of DEC,[4] Compaq was acquired for US$25 billion by HP in 2002"
J.E. Seagram,,,,Seagram was sold to French conglomerate Vivendi in 2000.
Rhone-Poulenc Rorer,217028,1325676,,In 1999 it merged with Hoechst AG to form Aventis.
Eaton,1490873,,,
Schering-Plough,310158,,"97, 98, 99, 00"," On November 4, 2009 Merck & Co. merged with Schering-Plough with the new company taking the name of Merck & Co."
Bethlehem Steel,11860,,,"After a decline in the American steel industry and other problems leading to the company's bankruptcy in 2001, the Bethlehem Steel Corporation was dissolved and the remaining assets sold to International Steel Group in 2003; Bethlehem Steel Corporation did not merge with/into International Steel Group."
FMC,37785,,"95, 2000,2001 ",
Navistar International,808450,,-,
VF,,,,
Avon Products,8868,,-,
American Standard,836102,,"95-97, 09-19 ",
Ingersoll-Rand,1466258,50485,43532,
Crown Holdings,1219601,,34001,
Cummins,26172,,,
Corning,24741,,1,
OfficeMax,929428,,"only have 97, 02, 03",
Pharmacia,12978,,"95, 96.00.01.02.14-19 ",
Owens-Illinois,812233,812074,,This may refer to OWENS & MINOR INC/VA/ or OWENS ILLINOIS INC /DE/
AMAX,,,,
Times Mirror,98349,925260,95. 99,Times Mirror Co. was acquired by the Tribune Company in 2000
Sun Microsystems,709519,,"95, 00, 01, 10-19","On April 20, 2009, it was announced that Oracle Corporation would acquire Sun for US$7.4 billion. The deal was completed on January 27, 2010.[3]"
Masco,62996,,,
Grumman,1133421,,34001,
Ryerson Tull,790528,1013595,"98-02, 15-18",
Gannett,39899,,,"In 2015, Gannett Co., Inc., spun off its publishing business into a separate publicly traded entity, while retaining the internet media divisions. Immediately following the spin off, the former parent Company (Gannett Co., Inc.) renamed itself Tegna and owns approximately 50 TV stations. The spun-off publishing business renamed itself ""Gannett"""
Pitney Bowes,78814,,"00,02 ",
Farmland Industries,34616,,, sold all of its assets in 2002–04
FINA,,,,
Kerr-McGee,55458,1141185,"95.96.01,07-19",
AMP,1242513,,"before 2009 and after 2014, 2012",
Agway,2852,,only have 94 and 02," On October 1, 2002 the company filed for Chapter 11 bankruptcy"
Air Products & Chem.,2969,,95-00,
Hershey Foods,47111,,,
Varity,63118,,only have 94 and 96,"In March 1999, LucasVarity was purchased by US automotive company TRW.[7]"
Rohm & Haas,84792,,43757, Dow Chemical Company bought Rohm and Haas for $15 billion in 2009
Tyco International,833444,,94-96,"On January 25, 2016, Johnson Controls announced it would merge with Tyco, and all businesses of Tyco and Johnson Controls would be combined under Tyco International plc, to be renamed as Johnson Controls International plc."
Union Camp,100783,,96,In 1999 it was acquired by International Paper.
Harris,202058,,"96, 97,01",
Maytag,63541,,01-02;07-19 ,The Maytag Corporation is an American home and commercial appliance brand owned by Whirlpool Corporation after the April 2006 acquisition of Maytag.
Berkshire Hathaway,109694,,99-2018,
Smurfit-Stone Container,94610,727742,,"SSCC was formed in November 1998, with the merger of Jefferson Smurfit Corporation (JSC) and Stone Container Corporation (Stone).I have also included the Smurfit Corporation cik here "
Universal,102037,,"95,97,98,00,01",
Ethyl,33656,,1,"In 2004, Ethyl Corporation became a subsidiary of NewMarket Corporation (NYSE: NEU)."
Premark International,800575,,missing 00-19 ,
Teledyne,1094285,,missing before 2002,
Seagate Technology,1137789,354952,,
Loral,1029850,1006269,,
Hercules,1280784,46989,," 2008, when it was merged into Ashland Inc."
Owens Corning,75234,1370946,,
Illinois Tool Works,49826,,,
Hormel Foods,48465,,,
PerkinElmer,31791,77551,,
Paccar,75362,,,
Sherwin-Williams,89800,,,
Pennzoil,77320,,"only have 94, 97, 98 ",
Temple-Inland,731939,,,
Readers Digest Assn.,858558,,,
Mapco,62142,,,
Avery Dennison,8818,,,
Diamond Shamrock,810316,,,
Ultramar Diamond Shamrock,887207,,,
Phelps Dodge,78066,,,
Land OLakes,1032562,,,
AMDAHL,4427,,, been a wholly owned subsidiary of Fujitsu since 1997.
Armstrong Holdings,1109304,,,
Baker Hughes,808362,1701605,,
Hasbro,46080,,,
Goodrich,42542,,,
Ball,9389,,,
Engelhard,352947,,,
Total Petroleum,,,,
Whitman,49573,1084230,,
Olin,74303,,,
Parker Hannifin,76334,,,
National Steel,70578,1231868,,
McDermott,708819,,,
Willamette Industries,107189,,,"In 2002, the lumber and paper company was purchased by competitor Weyerhaeuser of Federal Way, Washington in a hostile buyout and merged into Weyerhaeuser's existing operations."
Becton Dickinson,10795,,,
Westvaco,106498,1159297,,
Knight-Ridder,205520,,,"bought by McClatchy on June 27, 2006"
Quantum Chemical,,,,
Dean Foods,931336,,,
Dover,29905,,,
Intl. Multifoods,51410,,,"*cant find M&A records on wiki, 2005 onwards data missing"
Conner Peripherals,792397,,,"In 1996, Conner Peripherals was acquired by Seagate."
Premcor,1159119,,,Premcor was acquired by Valero in 2005. 
Maxxam,63814,,,
Manville,355473,,,
Brunswick,14930,,,
Collins & Aikman,1037123,846815,,
Stanley Works,93556,,,
Louisiana-Pacific,60519,,,
Polaroid,79326,1227728,," Polaroid Corporation was declared bankrupt in 2001, its brand and assets were sold off. The ""new"" Polaroid formed as a result, and itself declared bankruptcy in 2008, resulting in a further sale and in the present-day Polaroid Corporation"
Tosco,74091,,,"Tosco merged with Phillips Petroleum in 2001. Phillips merged with Conoco in 2002 to become ConocoPhillips, who spun off the Circle K stores to Canadian-based Alimentation Couche-Tard."
Tribune,726513,,,
E-SYSTEMS,,,,"In 1995, Raytheon Company acquired E-Systems, Inc."
ARMCO,7383,,,"In 1999, AK steel holding acquired Armco Inc., its former parent company, for $1.3 billio"
Burlington Industries Equity,870213,,,Its assets were acquired by International Textile Group (ITG) out of bankruptcy in late 2003
Tandem Computers,315180,,,
McGraw-Hill,64040,,,
Springs Industries,93102,,,"On June 27, 2007 Springs said that after 120 years, Springs would end manufacturing in South Carolina with the closing of its Grace and Close plants. The state would still have about 700 employees, most of them at distribution centers in Lancaster and Fort Lawn, and at the Fort Mill offices"
Molson Coors Brewing,24545,,,
Dow Corning,29917,,,"Following the December 11, 2015 announcement that it would merge with DuPont, "
York International,842662,,,"The York brand has been owned since August 2005 by Johnson Controls, when it was sold to them for $3.2 billion."
GenCorp,40888,,,
Asarco,7649,,,"In 1999 it was acquired by Grupo México, which had begun as Asarco's 49%-owned Mexican subsidiary in 1965."
Morton International,1035972,,,
Wang Laboratories,,,,"10-k available online, but somehow not in directory"
Central Soya,,,,
Arvin Industries,7636,,,
Pet,888455,,,missing 10-ks from 2007 onwards
Mattel,63276,,,
MID-AMERICA DAIRYMEN,789868,,,
Sequa,95301,,,
Fruit of the Loom,1053303,,,
Sonoco Products,91767,,,
Dow Jones,29924,,,2007 when an extended takeover battle saw News Corp take control of the company
Rubbermaid,814453,85627,,
Echlin,31348,,,"data available till 1996, no info traceable on internet"
USG,757011,,,
CENEX,823277,,,
New York Times,71691,,,
Shaw Industries,89498,,,"On January 4, 2001, under the guidance of CEO and President W. Norris Little, Sr. and CEO Bob Shaw, Shaw Industries was sold to Berkshire Hathaway Inc."
Witco,107889,,,
National Semiconductor,70530,,,
Imcera Group,,,,https://en.wikipedia.org/wiki/Mallinckrodt
Bausch & Lomb,10427,,,
Clorox,21076,,,
Sundstrand,95395,,,"Hamilton Sanstrand company was formed from the merger of Hamilton Standard and Sundstrand Corporation in 1999. In 2012, Hamilton Sundstrand was merged with Goodrich Corporation to form UTC Aerospace Systems. No evidence of Hamilton Sanstrand can be found in the directory "
Aeroquip-Vickers,59198,,,"On February 1, 1999, Eaton and Aeroquip-Vickers jointly announced that Eaton would acquire all of the outstanding common shares of Aeroquip-Vickers for $58 per share in cash, or approximately $1.7 billion."
Murphy Oil,717423,,,
Metaldyne,745448,1616817,,"Metaldyne Performance Group Inc.'s majority owner, American Securities LLC, and its affiliates acquired HHI Holdings in October 2012; Metaldyne, LLC in December 2012"
Burlington Resources,,,,
Freeport-McMoran,,,,
Cyprus Amax Minerals,,,,
Timken,,,,
National Service Industries,,,,
Harsco,,,,
General Signal,,,,
Nucor,,,,
Duracell International,,,,
Fleetwood Enterprises,,,,
Storage Technology,,,,
Newell Rubbermaid,,,,
Crown Central,,,,
American Greetings,,,,
Cabot,,,,
Lubrizol,,,,
Reliance Electric,,,,
Deluxe,,,,
Advanced Micro Devices,,,,
Lafarge,,,,
WestPoint Stevens,,,,
Great Lakes Chemical,,,,
Bowater,,,,
Nacco Industries,,,,
McCormick,,,,
Furniture Brands Intl.,,,,
Washington Post,,,,
Federal Paper Board,,,,
Hillenbrand Industries,,,,
Del Monte Foods,,,,
Lear,,,,
Joy Global,,,,
Nalco Chemical,,,,
Coltec Industries,,,,
Walter Industries,,,,
M.A. Hanna,,,,
Potlatch,,,,
Thiokol,,,,
Oryx Energy,,,,
Gold Kist Holdings,,,,
Crane,,,,
Wm. Wrigley Jr.,,,,
Great American Mgmt. & Inv.,,,,
Tektronix,,,,
Raychem,,,,
Dresser-Rand,,,,
Gerber Products,,,,
Varian Associates,,,,
Tecumseh Products,,,,
Rohr,,,,
My codes:
import csv
import os
import shutil
with open('missingfiles1.csv', 'r') as f:
os.chdir('/Users/lucy/Desktop/summer/datacollection/10-X_C_1993-2000/1994/QTR1')
reader=csv.reader(f)
company=()
for row in reader:
company[row[0]]={'cik':row[1]}
for f in oslistdir():
filename, filetype = os.path.splitext(f)
fn = filename.split('-')
if cik==fn[0]:
os.path.join('/Users/lucy/Desktop/summer/Summarydatafile',company)
shutil.move(f,os.path)
os.chdir('/Users/lucy/Desktop/summer/datacollection/missingfiles.csv')
I had the same issue but I am using package pandas.
Import pandas as pd
.
.
.
.
df = pd.read_csv('C:/Users/melissa/Documents/APIlist.csv', header=0, encoding='unicode_escape')
Adding the 'header=0' and encode the bytes to 'unicode escape' did the trick.

Altering JSON Structure

So I am using a webscraper to pull information on sneakers from a website. The son data that comes back is structured like so
[
{
"web-scraper-order": "1554084909-97",
"web-scraper-start-url": "https://www.goat.com/sneakers",
"productlink": "$200AIR JORDAN 6 RETRO 'INFRARED' 2019",
"productlink-href": "https://www.goat.com/sneakers/air-jordan-6-retro-black-infrared-384664-060",
"name": "Air Jordan 6 Retro 'Infrared' 2019",
"price": "Buy New - $200",
"description": "The 2019 edition of the Air Jordan 6 Retro ‘Infrared’ is true to the original colorway, which Michael Jordan wore when he captured his first NBA title. Dressed primarily in black nubuck with a reflective 3M layer underneath, the mid-top features Infrared accents on the midsole, heel tab and lace lock. Nike Air branding adorns the heel and sockliner, an OG detail last seen on the 2000 retro.",
"releasedate": "2019-02-16",
"colorway": "Black/Infrared 23-Black",
"brand": "Air Jordan",
"designer": "Tinker Hatfield",
"technology": "Air",
"maincolor": "Black",
"silhouette": "Air Jordan 6",
"nickname": "Infrared",
"category": "lifestyle",
"image-src": "https://image.goat.com/crop/1250/attachments/product_template_additional_pictures/images/018/675/318/original/464372_01.jpg.jpeg"
},
{
"web-scraper-order": "1554084922-147",
"web-scraper-start-url": "https://www.goat.com/sneakers",
"productlink": "$190YEEZY BOOST 350 V2 'CREAM WHITE / TRIPLE WHITE'",
"productlink-href": "https://www.goat.com/sneakers/yeezy-boost-350-v2-cream-white-cp9366",
"name": "Yeezy Boost 350 V2 'Cream White / Triple White'",
"price": "Buy New - $220",
"description": "First released on April 29, 2017, the Yeezy Boost 350 V2 ‘Cream White’ combines a cream Primeknit upper with tonal cream SPLY 350 branding, and a translucent white midsole housing full-length Boost. Released again in October 2018, this retro helped fulfill Kanye West’s oft-repeated ‘YEEZYs for everyone’ Twitter mantra, as adidas organized the biggest drop in Yeezy history by promising pre-sale to anyone who signed up on the website. Similar to the first release, the ‘Triple White’ 2018 model features a Primeknit upper, a Boost midsole and custom adidas and Yeezy co-branding on the insole.",
"releasedate": "2017-04-29",
"colorway": "Cream White/Cream White/Core White",
"brand": "adidas",
"designer": "Kanye West",
"technology": "Boost",
"maincolor": "White",
"silhouette": "Yeezy Boost 350",
"nickname": "Cream White / Triple White",
"category": "lifestyle",
"image-src": "https://image.goat.com/crop/1250/attachments/product_template_additional_pictures/images/014/822/695/original/116662_03.jpg.jpeg"
},
However, I want to change it so that the top level node is sneakers and the next level down would be a specific sneaker brand ( Jordan, Nike, Adidas) and then the list of sneakers that belong to that brand. So my JSON structure would look something like this
Sneakers {
Adidas :{
[shoe1,
shoe2,
....
] },
Jordan: {
[shoe1,
shoe2,
....
]
}
}
I am not sure what tool I could use to do that. Any help would be greatly appreciated. All I have at the moment is the JSON file and it is not in the structure that I want it to be in.
One way of doing this would be to populate a dict whose keys are brand names and their values are lists of sneaker records. Assuming that data is your original list, here's the code:
sneakers_by_brand = {}
for record in data:
if sneakers_by_brand.get(record.get("brand")):
sneakers_by_brand[record.get("brand")].append(record)
else:
sneakers_by_brand[record.get("brand")] = [record]
print(sneakers_by_brand)

How do I insert spatial data into a table contains road column

I am getting this error message while I am trying to insert my data into line table, using coordinates for points between road segments.
ERROR: parse error - invalid geometry
HINT: "SRID=27700;LINESTRING((5" <-- parse error at position 24 within geometry
SQL state: XX000
Part of my code:
NSERT INTO public."RoadSegments"("no", "seg_ID", "description", "location", "length", "the_geom")
VALUES
(1,'Seg_1','Shephards Bush to Royal Crescent','Shephards Bush','540',ST_GeomFromEWKT('SRID=27700;LINESTRING((51.504593 -0.220437),(51.505233 -0.214105))')),
(2,'Seg_2','Royal Crescent to Norland Square','Notting Hill','306',ST_GeomFromEWKT('SRID=27700;LINESTRING((51.505233 -0.214105),(51.506053 -0.209956))')),
(3,'Seg_3','Norland Square to Holland Park','Notting Hill','383',ST_GeomFromEWKT('SRID=27700;LINESTRING((51.506053 -0.209956),(51.507575 -0.204795))')),
(4,'Seg_4','Holland Park to Notting Hill Gate','Notting Hill','477',ST_GeomFromEWKT('SRID=27700;LINESTRING((51.507575 -0.204795),(51
You seem to have too much parentheses in your LINESTRING parameters. Try the following:
INSERT INTO public."RoadSegments"("no", "seg_ID", "description", "location", "length", "the_geom")
VALUES
(1,'Seg_1','Shephards Bush to Royal Crescent', 'Shephards Bush','540',ST_GeomFromEWKT('SRID=27700;LINESTRING(51.504593 -0.220437,51.505233 -0.214105)')),
(2,'Seg_2','Royal Crescent to Norland Square', 'Notting Hill', '306',ST_GeomFromEWKT('SRID=27700;LINESTRING(51.505233 -0.214105,51.506053 -0.209956)')),
(3,'Seg_3','Norland Square to Holland Park', 'Notting Hill', '383',ST_GeomFromEWKT('SRID=27700;LINESTRING(51.506053 -0.209956,51.507575 -0.204795)')),
(4,'Seg_4','Holland Park to Notting Hill Gate','Notting Hill', '477',ST_GeomFromEWKT('SRID=27700;LINESTRING(51.507575 -0.204795,51 ... '))
This should be the correct syntax (no extra parentheses around the coordinates)

how can i extract special kind of table from website in perl?

I am trying to fetch all tables from the website http://finance.yahoo.com/etf/lists/?bypass=true&mod_id=mediaquotesetf&tab=tab1&scol=imkt&stype=desc&rcnt=50&page=1, using Perl module HTML::TableExtract, but I can't get the desired table; instead I get the first two tables only, which are useless to me.
Here is my code:
#!/usr/bin/perl
#!perl -w
use DBI;
use strict;
use WWW::Mechanize;
use HTML::TableExtract;
my $mech= WWW::Mechanize->new();
my $url= 'http://finance.yahoo.com/etf/lists/?bypass=true&mod_id=mediaquotesetf&tab=tab1&scol=imkt&stype=desc&rcnt=50&page=1';
$mech -> get($url);
chomp(my $script = $mech -> content);
my $table=new HTML::TableExtract();
$table->parse($script);
foreach my $ts($table->tables){
print "Table (", join(',', $ts->coords), "):\n";
foreach my $row ($ts->rows){
print join(',', #$row), "\n";
}
}
output:
Table (0,0):
,Search FinanceSearch Web
Table (0,1):
Quotes you view appear here for quick access.
like this I only get the first two tables instead of all of them.
The third table is generated dynamically using JavaScript. WWW::Mechanize doesn't support JavaScript, and you will need to use WWW::Mechanize::Firefox instead
Note that this will require you to install a Firefox web browser, and its mozrepl plugin, as well as the MozRepl Perl module
use strict;
use warnings 'all';
use feature 'say';
use open qw/ :std :encoding(UTF-8) /;
use WWW::Mechanize::Firefox;
use HTML::TableExtract;
use constant URL => 'http://finance.yahoo.com/etf/lists/?bypass=true&mod_id=mediaquotesetf&tab=tab1&scol=imkt&stype=desc&rcnt=50&page=1';
my $mech = WWW::Mechanize::Firefox->new;
$mech->autoclose_tab(0);
$mech->get(URL);
my $te = HTML::TableExtract->new(depth => 0, count => 2);
$te->parse( $mech->content );
for my $row ( $te->rows ) {
local $" = ',';
print "#$row\n";
}
output
ETF Name,Ticker,Category,Fund Family,Intraday Return,3-MO Return,YTD Return,1-YR Return,3-YR Return,5-YR Return
UBS ETRACS ISE Exclusively Hmbldrs ETN,HOMX,Consumer Cyclical,UBS Group AG,+13.43%,-6.74%,-6.74%,-19.54%,0.0%,0.0%
VelocityShares 3x Long Natural Gas ETN,UGAZ,Trading-Leveraged Commodities,Credit Suisse AG,+9.33%,-60.15%,-60.15%,-91.36%,-81.58%,0.0%
ProShares Ultra Bloomberg Natural Gas,BOIL,Trading-Leveraged Commodities,ProShares,+5.96%,-41.13%,-41.13%,-76.12%,-62.1%,0.0%
Direxion Daily Brazil Bull 3X ETF,BRZU,Trading-Leveraged Equity,Direxion Funds,+4.24%,+66.64%,+66.64%,-61.7%,0.0%,0.0%
DB Commodity Double Long ETN,DYY,Trading-Leveraged Commodities,Deutsche Bank AG,+4.16%,-25.87%,-25.87%,-41.98%,-34.91%,-32.8%
Deutsche X-trackers MSCI EMktsHiDvYdHgEq,HDEE,Diversified Emerging Mkts,Deutsche Asset Management,+3.73%,-2.15%,-2.15%,0.0%,0.0%,0.0%
DB Agriculture Double Long ETN,DAG,Trading-Leveraged Commodities,Deutsche Bank AG,+3.57%,+3.12%,+3.12%,-19.14%,-29.52%,-25.49%
United States Natural Gas,UNG,Commodities Energy,United States Commodity Funds LLC,+3.15%,-23.18%,-23.18%,-49.7%,-32.73%,-32.09%
Direxion Daily Jr Gld Mnrs Bear 3X ETF,JDST,Trading-Inverse Equity,Direxion Funds,+3.03%,-80.83%,-80.83%,-88.33%,0.0%,0.0%
Direxion Daily S&P Biotech Bull 3X ETF,LABU,Trading-Leveraged Equity,Direxion Funds,+2.97%,-67.51%,-67.51%,0.0%,0.0%,0.0%
VelocityShares 3x Inverse Silver ETN,DSLV,Trading-Inverse Commodities,Credit Suisse AG,+2.88%,-36.11%,-36.11%,-15.03%,+13.64%,0.0%
ProShares Ultra MSCI Brazil Capped,UBR,Trading-Leveraged Equity,ProShares,+2.85%,+51.79%,+51.79%,-37.19%,-42.04%,-38.48%
Direxion Daily India Bull 3X ETF,INDL,Trading-Leveraged Equity,Direxion Funds,+2.71%,-9.82%,-9.82%,-48.84%,-13.11%,-22.69%
Direxion Daily Real Estate Bull 3X ETF,DRN,Trading-Leveraged Equity,Direxion Funds,+2.66%,+15.65%,+15.65%,-0.71%,+21.8%,+22.37%
iShares US Telecommunications,IYZ,Communications,iShares,+2.63%,+7.43%,+7.43%,+3.79%,+10.74%,+7.88%
ProShares Ultra Semiconductors,USD,Trading-Leveraged Equity,ProShares,+2.58%,-4.09%,-4.09%,-6.65%,+31.6%,+15.12%
Direxion Daily Pharmctcl&Medcl Bl 2X ETF,PILL,Trading-Leveraged Equity,Direxion Funds,+2.57%,-27.81%,-27.81%,0.0%,0.0%,0.0%
IQ Hedge Event-Driven Tracker ETF,QED,Market Neutral,IndexIQ,+2.54%,+1.51%,+1.51%,-1.64%,0.0%,0.0%
Direxion Daily Regional Bnks Bull 3X ETF,DPST,Trading-Leveraged Equity,Direxion Funds,+2.51%,-20.73%,-20.73%,0.0%,0.0%,0.0%
VelocityShares 3x Inverse Gold ETN,DGLD,Trading-Inverse Commodities,Credit Suisse AG,+2.44%,-40.19%,-40.19%,-24.19%,+6.6%,0.0%
Direxion Daily South Korea Bull 3X ETF,KORU,Trading-Leveraged Equity,Direxion Funds,+2.43%,+12.48%,+12.48%,-30.37%,0.0%,0.0%
VelocityShares Daily Inverse VIX ST ETN,XIV,Volatility,Credit Suisse AG,+2.43%,+0.31%,+0.31%,-25.29%,+3.55%,+13.26%
ProShares Ultra S&P Regional Banking,KRU,Trading-Leveraged Equity,ProShares,+2.42%,-22.89%,-22.89%,-17.41%,+10.48%,+9.61%
Global X FTSE Andean 40 ETF,AND,Latin America Stock,Global X Funds,+2.38%,+12.8%,+12.8%,-15.28%,-19.56%,-11.0%
AccuShares Spot CBOE® VIX® ETC Down,VXDN,Volatility,AccuShares™,+2.32%,-12.72%,-12.72%,0.0%,0.0%,0.0%
ProShares Short S&P Regional Banking,KRS,Trading-Inverse Equity,ProShares,+2.3%,+7.46%,+7.46%,-0.5%,-12.4%,-14.52%
United States 12 Month Natural Gas,UNL,Commodities Energy,United States Commodity Funds LLC,+2.3%,-8.84%,-8.84%,-29.88%,-22.73%,-23.91%
ProShares Short VIX Short-Term Futures,SVXY,Volatility,ProShares,+2.28%,+0.16%,+0.16%,-25.73%,+3.53%,0.0%
SPDR® S&P Transportation ETF,XTN,Industrials,SPDR State Street Global Advisors,+2.24%,+7.3%,+7.3%,-12.73%,+12.63%,+12.34%
iShares MSCI UAE Capped,UAE,Miscellaneous Region,iShares,+2.22%,+5.31%,+5.31%,-4.98%,0.0%,0.0%
VelocityShares 3x Inverse Crude Oil ETN,DWTI,Trading-Inverse Commodities,Credit Suisse AG,+2.21%,-20.19%,-20.19%,+19.62%,+56.72%,0.0%
ProShares Ultra High Yield,UJB,Trading-Leveraged Debt,ProShares,+2.19%,+10.09%,+10.09%,-6.81%,+1.36%,0.0%
iPath® Bloomberg Livestock SubTR ETN,COW,Commodities Agriculture,Barclays Funds,+2.19%,+0.94%,+0.94%,-10.57%,-3.1%,-5.92%
iPath® Bloomberg Natural Gas SubTR ETN,GAZ,Commodities Energy,Barclays Funds,+2.19%,-31.94%,-31.94%,-59.17%,-44.91%,-49.91%
Direxion Daily Small Cap Bull 3X ETF,TNA,Trading-Leveraged Equity,Direxion Funds,+2.16%,-8.7%,-8.7%,-35.41%,+10.14%,+6.17%
ProShares Ultra Utilities,UPW,Trading-Leveraged Equity,ProShares,+2.16%,+30.83%,+30.83%,+28.99%,+22.85%,+24.31%
SPDR® Wells Fargo Preferred Stock ETF,PSK,Preferred Stock,SPDR State Street Global Advisors,+2.14%,+1.77%,+1.77%,+5.83%,+5.83%,+6.08%
ProShares UltraShort Silver,ZSL,Trading-Inverse Commodities,ProShares,+2.14%,-23.44%,-23.44%,-1.99%,+21.56%,-2.97%
ProShares UltraPro Russell2000,URTY,Trading-Leveraged Equity,ProShares,+2.14%,-8.53%,-8.53%,-34.81%,+10.76%,+7.06%
Direxion Daily Emrg Mkts Bull 3X ETF,EDC,Trading-Leveraged Equity,Direxion Funds,+2.11%,+13.64%,+13.64%,-44.79%,-25.84%,-28.1%
DB Agriculture Short ETN,ADZ,Trading-Inverse Commodities,Deutsche Bank AG,+2.1%,-11.59%,-11.59%,-5.04%,+11.94%,+7.62%
DB 3x Long 25+ Year Treasury Bond ETN,LBND,Trading-Leveraged Debt,Deutsche Bank AG,+2.08%,+22.67%,+22.67%,+2.15%,+11.31%,+24.21%
PureFunds ISE Cyber Security™ ETF,HACK,Technology,Pure Funds,+2.02%,-7.45%,-7.45%,-14.3%,0.0%,0.0%
ProShares UltraPro MidCap400,UMDD,Trading-Leveraged Equity,ProShares,+1.95%,+6.04%,+6.04%,-19.63%,+20.32%,+16.2%
ProShares Ultra Telecommunications,LTL,Trading-Leveraged Equity,ProShares,+1.79%,+13.74%,+13.74%,+0.87%,+18.18%,+11.67%
Direxion Daily Hmbldrs&Supls Bull 3X ETF,NAIL,Trading-Leveraged Equity,Direxion Funds,+1.79%,-10.32%,-10.32%,0.0%,0.0%,0.0%
Teucrium Wheat ETF,WEAT,Commodities Agriculture,Teucrium,+1.78%,-1.54%,-1.54%,-17.67%,-21.21%,0.0%
Vanguard Telecommunication Services ETF,VOX,Communications,Vanguard,+1.76%,+11.11%,+11.11%,+11.82%,+11.64%,+9.99%
Direxion Daily Financial Bull 3X ETF,FAS,Trading-Leveraged Equity,Direxion Funds,+1.75%,-14.79%,-14.79%,-18.93%,+21.63%,+14.48%
US Global Jets ETF,JETS,Miscellaneous Sector,U.S. Global Investors,+1.75%,+1.77%,+1.77%,0.0%,0.0%,0.0%