Hello,
I am using web API to fetch the data from a data source and it works fine on desktop, but after configuring the incremental refresh on a date/time field and published to service, incremental refresh does not work there. There are 100+ million rows and the manual refresh started to fail early this week after it hit the 5 hr limit in the premium workspace. It is mandatory for the incremental refresh to set up but it does not seem to work, eagerly looking for any suggestions from the community folks... I have pasted my M code below for reference.
let
url = "https://abc.domain.com",
part = "/data/xAPI/statements?since=2020-10-05",
FnGetOnePage =
(part) as record =>
let
Source = Json.Document(Web.Contents("https://abc.domain.com",[RelativePath=part,
Headers=[Authorization="Basic YzY3Nzc5MmMxZDY0YTE4ZmMzOWJkZTANmExNjE2ZWU0MjE5NjozYzY0OTQ5ZTQxYmEzZjA3MWY5ZGE1MTliNWY5NDVhMzBlNmI2MWUw",#"X-Experience-API-Version"="1.0.3"]])),
data = try Source[statements] otherwise null,
next = try Source[more] otherwise null,
Statements = [Data=data, Next=next]
in
Statements,
GeneratedList =
List.Generate(
()=>[i=0, Statements = FnGetOnePage(part)],
each [Statements][Data]<>null,
each [i=[i]+1, Statements = FnGetOnePage([Statements][Next])],
each [Statements][Data]),
#"Converted to Table" = Table.FromList(GeneratedList, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
#"Expanded Column1" = Table.ExpandListColumn(#"Converted to Table", "Column1"),
#"Expanded Column2" = Table.ExpandRecordColumn(#"Expanded Column1", "Column1", {"actor", "verb", "context", "timestamp", "object", "id", "stored", "authority", "result"}, {"actor", "verb", "context", "timestamp", "object", "id", "stored", "authority", "result"}),
#"Expanded actor" = Table.ExpandRecordColumn(#"Expanded Column2", "actor", {"name", "mbox"}, {"name", "mbox"}),
#"Expanded verb" = Table.ExpandRecordColumn(#"Expanded actor", "verb", {"display"}, {"display"}),
#"Renamed to UPI" = Table.RenameColumns(#"Expanded verb",{{"name", "UPI"}}),
#"Expanded display" = Table.ExpandRecordColumn(#"Renamed to UPI", "display", {"en-US"}, {"en-US"}),.............
