Foundry - Using tranforms verbs inmemory datastore to test incremental transforms - palantir-foundry

Question

Im using the in-memory datastore approach to test incremental transform and im receiving the error below. Any idea what i might be doing wrong.
def test_transformdata_incr(spark_session):
df_input = spark_session.createDataFrame([
(1, 'abc', 1000, '2022-08-01'),
(2, 'abc', 2000, '2022-08-02'),
(3, 'def', 2000, '2022-08-01')
], ['a', 'b', 'c', 'd'])
df_expected = spark_session.createDataFrame([
('abc', 3000, '2022-08-12 15:10:00'),
('def', 2000, '2022-08-12 15:10:00')
], ['b', 's_colc', 'ts'])
transform_input = '/data/sampledata'
transform_output = '/data/sampledata_output'
pipeline = Pipeline()
pipeline.add_transforms(compute)
store = InMemoryDatastore()
store.store_dataframe(transform_input, df_input)
runner = TransformRunner(pipeline, datastore=store)
df_output = runner.build_dataset(spark_session, transform_output)
assert df_output.subtract(df_expected).count() == 0
assert df_expected.subtract(df_output).count() == 0
assert df_output.schema == df_expected.schema
Error:
df_output = runner.build_dataset(spark_session, transform_output)
myproject/tests/test_transformdata_incr.py:30:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ ../build/conda/env/lib/python3.8/site-packages/transforms/verbs/testing/TransformRunner.py:74:
in build_dataset
self._build_dataset_recursive(spark_session, alias) ../build/conda/env/lib/python3.8/site-packages/transforms/verbs/testing/TransformRunner.py:84:
in _build_dataset_recursive
self._build_derived(spark_session, alias) ../build/conda/env/lib/python3.8/site-packages/transforms/verbs/testing/TransformRunner.py:107:
in _build_derived
transform.compute(**kwargs)
self = <transforms.api._incremental._IncrementalCompute object at
0x7f1b6f9afd60> ctx =
<transforms.verbs.testing.dummies.DummyTransformContext object at
0x7f1b6e85c790> transform_ios = {'output':
<transforms.verbs.testing.dummies.DummyTransformOutput object at
0x7f1b6e85c730>, 'source_df':
<transforms.verbs.testing.dummies.DummyTransformInput object at
0x7f1b6e85c070>} tinputs = {'source_df':
<transforms.verbs.testing.dummies.DummyTransformInput object at
0x7f1b6e85c070>} toutputs = {} parameters = {'output':
<transforms.verbs.testing.dummies.DummyTransformOutput object at
0x7f1b6e85c730>}
def __call__(self, ctx=None, **transform_ios): # pylint: disable=arguments-differ
"""Run the computation by dynamically constructing IncrementalX objects from the general X objects.
TransformInput -> IncrementalTransformInput
TransformOutput -> IncrementalTransformOutput
TransformContext -> IncrementalTransformContext
"""
tinputs = {
name: tio for name, tio in transform_ios.items()
if isinstance(tio, _transform.TransformInput) and not isinstance(tio, _transform.TransformOutput)
}
toutputs = {
name: tio for name, tio in transform_ios.items() if isinstance(tio, _transform.TransformOutput)
}
parameters = {
name: param for name, param in transform_ios.items()
if not isinstance(param, _transform.TransformInput) and not isinstance(param, _transform.TransformOutput)
}
foundry = list(toutputs.values())[0]._dfreader._foundry # Kinda horrible, but we grab a foundry instance E IndexError:
list index out of range
../build/conda/env/lib/python3.8/site-packages/transforms/api/_incremental.py:169:
IndexError

Answer 1

I have found a work-around.
you can mock the decorator:
def _fake_incremental_decorator(trasnform_obj):
DummyTransformContext.is_incremental = False
return trasnform_obj
You can use then a context manager within the test to apply the mock:
with patch('transforms.api.incremental', return_value=_fake_incremental_decorator):
pipeline.add_transforms(compute)