diff --git a/graph/component/base.py b/graph/component/base.py index b424d226f..06574e32d 100644 --- a/graph/component/base.py +++ b/graph/component/base.py @@ -470,7 +470,10 @@ class ComponentBase(ABC): break if upstream_outs: - return pd.concat(upstream_outs, ignore_index=False) + df = pd.concat(upstream_outs, ignore_index=True) + if "content" in df: + df = df.drop_duplicates(subset=['content']).reset_index(drop=True) + return df return pd.DataFrame() def get_stream_input(self):