Sheet Script to Dedupe Rows: Unexpected Behavior - google-apps-script

Question

I have a script which posts data from one sheet to another and then de-dupes the data. The dedupe script appears to only dedupe SOME of the data. For instance, rows 2-13 remain duplicates while rows after row 35 are deduped. Rows at the top are posted from the insert script.
How can I get the script to:
a) Dedupe the full range of data.
b) Remove the new duplicate rows posted from the insert script rather than rows that were already there.
I've tried setting the full range of the sheet to be checked, but this seems to interfere with the post to the new data. This is the simplest script I've managed to find/develop.
function Run(){
insert();
dedupe();
}
function insert() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var source = ss.getSheetByName('Candidate Refresh'); // change here
var des = ss.getSheetByName('Candidate Listing'); // change here
var sv = source
.getDataRange()
.getValues();
sv.shift();
des.insertRowsAfter(1, sv.length);
des.getRange(2, 1, sv.length, source.getLastColumn()).setValues(sv);
}
function dedupe() {
var sheet = SpreadsheetApp.getActive().getSheetByName('Candidate Listing');
var data = sheet.getDataRange().getValues();
var newData = new Array();
for(i in data){
var row = data[i];
var duplicate = false;
for(j in newData){
if(row.join() == newData[j].join()){
duplicate = true;
}
}
if(!duplicate){
newData.push(row);
}
}
sheet.clearContents();
sheet.getRange(1, 1, newData.length,
newData[0].length).setValues(newData);
}
I would expect new data to be posted, the full range of data to be checked for duplicates, and new data posted by the script to be removed if it is found to be a duplicate.

Answer 1

So I think my original post did work from the de-dupe perspective but I also needed it to only look at one value so I found the below script to do so which works much better.
function removeDuplicates() {
var sheet = SpreadsheetApp.getActiveSheet();
var rows = sheet.getLastRow();
var firstColumn = sheet.getRange(1, 2, rows, 1).getValues();
firstColumn = firstColumn.map(function (e) {return e[0]})
for (var i = rows; i >0; i--) {
if (firstColumn.indexOf(firstColumn[i-1]) != i-1) {
sheet.deleteRow(i);
}
}
}