SLC code not filling all Landsat 7 SR gaps - json

Question

I am a beginner in Google Earth Engine code and am trying to apply the SLC-gap code to Landsat 7 Surface Reflectance images. Using resources available on StackOverflow, I generated the code below; however, when I bring the images into QGIS, there still appear to be gaps. Is my code incorrect or did I not properly apply it to the images?
First, I masked the clouds based on the pixel_qa band of Landsat SR data:
var cloudMaskL7 = function(image) {
var qa = image.select('pixel_qa');
var cloud = qa.bitwiseAnd(1 << 5)
.and(qa.bitwiseAnd(1 << 7))
.or(qa.bitwiseAnd(1 << 3));
Then, I removed edge pixels that don't occur in all bands:
var mask2 = image.mask().reduce(ee.Reducer.min());
return image.updateMask(cloud.not()).updateMask(mask2);
};
var l7 = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR')
.filterDate('2004-09-15', '2004-12-31')
.map(cloudMaskL7);
var visParams = {
bands: ['B3', 'B2', 'B1'],
min: 0,
max: 3000,
gamma: 1.4,
};
Map.setCenter(36.197, 31.701,7);
Map.addLayer(l7.median(), visParams);
Then, I mapped the function over one year of Landsat 7 TOA data and took the median and mapped it for Jordan.
var composite = l7.map(cloudMaskL7)
.median();
Map.setCenter(36.124, 31.663);
Map.addLayer(composite, {bands: ['B4', 'B3', 'B2'], max: 0.3});
Then, I tried to fill the SLC Landsat 7 gaps by applying the USGS L7 Phase-2 Gap filling protocol, using a single kernel size.
var MIN_SCALE = 1/3;
var MAX_SCALE = 3;
var MIN_NEIGHBORS = 144;
var GapFill = function(src, fill, kernelSize) {
var kernel = ee.Kernel.square(kernelSize * 30, 'meters', false);
var common = src.mask().and(fill.mask());
var fc = fill.updateMask(common);
var sc = src.updateMask(common);
Then, I found the primary scaling factors with a regression and interleaved the bands for the regression (assumes the bands have the same names).
var regress = fc.addBands(sc);
regress = regress.select(regress.bandNames().sort());
var fit = regress.reduceNeighborhood(ee.Reducer.linearFit().forEach(src.bandNames()), kernel, null, false);
var offset = fit.select('.*_offset');
var scale = fit.select('.*_scale');
Then, I found the secondary scaling factors using just means and stddev.
var reducer = ee.Reducer.mean().combine(ee.Reducer.stdDev(), null, true);
var src_stats = src.reduceNeighborhood(reducer, kernel, null, false);
var fill_stats = fill.reduceNeighborhood(reducer, kernel, null, false);
var scale2 = src_stats.select('.*stdDev').divide(fill_stats.select('.*stdDev'));
var offset2 = src_stats.select('.*mean').subtract(fill_stats.select('.*mean').multiply(scale2));
var invalid = scale.lt(MIN_SCALE).or(scale.gt(MAX_SCALE));
scale = scale.where(invalid, scale2);
offset = offset.where(invalid, offset2);
I applied the scaling and mask off pixels that didn't have enough neighbors.
var count = common.reduceNeighborhood(ee.Reducer.count(), kernel, null, true, 'boxcar');
var scaled = fill.multiply(scale).add(offset)
.updateMask(count.gte(MIN_NEIGHBORS));
return src.unmask(scaled, true);
};
var source = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR');
var fill = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR');
I loaded a table of boundaries and filter.
var Jordan = ee.FeatureCollection('USDOS/LSIB_SIMPLE/2017')
.filter(ee.Filter.or(
ee.Filter.eq('country_co', 'JO')));
var clippedJordan = composite.clipToCollection(Jordan);
I displayed the results for Jordan; however, the SLC gaps appear not to be filled. I proceed to calculate the MSAVI2 values using these images, so the remaining gaps influence the results.
var mc = Map.setCenter(36.274, 31.682, 6);
var visParams = {bands: ['B3', 'B2', 'B1']};
Map.addLayer(clippedJordan, visParams, 'clipped composite');
Any advice would be greatly appreciated!

Answer 1

I am not up on the latest Landsat 7 gap-filling technology for SLC-off imagery, but here is a greatly simplified version of what you were trying to do. I removed a lot of (unnecessary?) stuff, increased the kernel size by a lot, and increased the timeframe over which the median replacement is generated. It might get close to what you want:
var cloudMaskL7 = function(image) {
var qa = image.select('pixel_qa');
var cloud = qa.bitwiseAnd(1 << 5)
.and(qa.bitwiseAnd(1 << 7))
.or(qa.bitwiseAnd(1 << 3));
var mask2 = image.mask().reduce(ee.Reducer.min());
return image.updateMask(cloud.not()).updateMask(mask2);
};
var l7 = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR')
.map(cloudMaskL7);
var kernelSize = 10;
var kernel = ee.Kernel.square(kernelSize * 30, 'meters', false);
var GapFill = function(image) {
var start = image.date().advance(-1, 'year');
var end = image.date().advance(1, 'year');
var fill = l7.filterDate(start, end).median();
var regress = fill.addBands(image);
regress = regress.select(regress.bandNames().sort());
var fit = regress.reduceNeighborhood(ee.Reducer.linearFit().forEach(image.bandNames()), kernel, null, false);
var offset = fit.select('.*_offset');
var scale = fit.select('.*_scale');
var scaled = fill.multiply(scale).add(offset);
return image.unmask(scaled, true);
};
// TESTING CODE
var point = ee.Geometry.Point(36.124, 31.663);
Map.centerObject(point, 11);
var check = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR')
.filterBounds(point)
.filterDate('2004-09-15', '2004-12-31');
var checkImage = ee.Image(check.first());
var visParams = {bands: ['B4', 'B3', 'B2'], min: 200, max: 5500};
Map.addLayer(checkImage, visParams, 'source');
// Test composite.
var checkStart = checkImage.date().advance(-1, 'year');
var checkEnd = checkImage.date().advance(1, 'year');
var composite = l7.filterDate(checkStart, checkEnd).median();
Map.addLayer(composite, visParams, 'median');
// Rough implementation for comparison.
var replaced = checkImage.unmask(composite);
Map.addLayer(replaced, visParams, 'simple');
// Fancy implementation.
var filled = ee.Image(check.map(GapFill).first());
Map.addLayer(filled, visParams, 'filled');
EDIT: The answer now shows how to map this over a collection. Watch out, because I don't know how well this will scale. If you decide to map it over a big area or long time series, you've been warned.