Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(reasoner): fix odps check type error #97

Merged
merged 2 commits into from
Jan 18, 2024
Merged

fix(reasoner): fix odps check type error #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b723efd
ignore odps type check for now
wangsff Jan 17, 2024
aa4bc41
add getAllEntityName function
wangsff Jan 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/table/FieldType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import com.antgroup.openspg.reasoner.common.types.KTDouble$;
import com.antgroup.openspg.reasoner.common.types.KTInteger$;
import com.antgroup.openspg.reasoner.common.types.KTLong$;
import com.antgroup.openspg.reasoner.common.types.KTObject$;
import com.antgroup.openspg.reasoner.common.types.KTString$;
import com.antgroup.openspg.reasoner.common.types.KgType;
import java.io.Serializable;
Expand All @@ -28,7 +29,7 @@ public enum FieldType implements Serializable {
DOUBLE(KTDouble$.MODULE$),
BOOLEAN(KTBoolean$.MODULE$),
UNKNOWN(KTString$.MODULE$),
;
OBJECT(KTObject$.MODULE$);

private final KgType kgType;

Expand Down
22 changes: 21 additions & 1 deletion .../kgdsl-parser/src/main/scala/com/antgroup/openspg/reasoner/parser/utils/ParserUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ package com.antgroup.openspg.reasoner.parser.utils
import com.antgroup.openspg.reasoner.common.Utils
import com.antgroup.openspg.reasoner.common.table.Field
import com.antgroup.openspg.reasoner.common.types.KTString
import com.antgroup.openspg.reasoner.lube.block.{Block, TableResultBlock}
import com.antgroup.openspg.reasoner.lube.block.{Block, MatchBlock, TableResultBlock}
import scala.collection.JavaConverters._
import scala.collection.mutable

object ParserUtils {

Expand All @@ -34,4 +36,22 @@ object ParserUtils {
}
}

def getAllEntityName(javaBlockList: java.util.List[Block]): java.util.Set[String] = {
val blockList: List[Block] = javaBlockList.asScala.toList
val entityNames = new mutable.HashSet[String]()
if (blockList != null && blockList.nonEmpty) {
blockList.foreach(block => {
block.transform[Unit] {
case (MatchBlock(_, patterns), _) =>
patterns.values
.map(_.graphPattern.nodes)
.flatMap(_.values)
.foreach(node => entityNames ++= node.typeNames)
case _ =>
}
})
}
entityNames.toSet.asJava
}

}
34 changes: 34 additions & 0 deletions ...sl-parser/src/test/scala/com/antgroup/openspg/reasoner/parser/utils/ParserUtilsTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@ package com.antgroup.openspg.reasoner.parser.utils

import com.antgroup.openspg.reasoner.common.constants.Constants
import com.antgroup.openspg.reasoner.common.table.FieldType
import com.antgroup.openspg.reasoner.lube.block.Block
import com.antgroup.openspg.reasoner.parser.OpenSPGDslParser
import org.scalatest.funspec.AnyFunSpec
import org.scalatest.matchers.should.Matchers.{convertToAnyShouldWrapper, equal}
import scala.collection.JavaConverters._

class ParserUtilsTest extends AnyFunSpec {

Expand Down Expand Up @@ -93,4 +95,36 @@ class ParserUtilsTest extends AnyFunSpec {
FieldType.STRING,
FieldType.STRING))
}
it("test getAllEntityName") {
// scalastyle:off
val dsl =
"""
|Define (s:User)-[p:belongTo]->(o:Crowd/`Men`) {
| GraphStructure {
| (evt:TradeEvent)-[pr:relateUser]->(s:User)
| }
| Rule{
| R1: s.sex == '男'
| R2: evt.statPriod in ['日', '月']
| DayliyAmount = group(s).if(evt.statPriod=='日').sum(evt.amount)
| MonthAmount = group(s).if(evt.statPriod=='月').sum(evt.amount)
| R3: DayliyAmount > 300
| R4: MonthAmount < 500
| R5: (R3 and R1) and (not(R4 and R1))
| }
|}
|GraphStructure {
| (a:Crowd/`Men`)
|}
|Rule {
|}
|Action {
| get(a.id)
|}
|""".stripMargin
val parser = new OpenSPGDslParser()
val blockList: List[Block] = parser.parseMultipleStatement(dsl)
val entityNameSet: Set[String] = ParserUtils.getAllEntityName(blockList.asJava).asScala.toSet
entityNameSet should equal(Set.apply("User", "TradeEvent", "Crowd/Men"))
}
}
4 changes: 3 additions & 1 deletion ...r/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/odps/OdpsUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,8 @@ public static boolean isSchemaMatch(OdpsTableInfo odpsTableInfo, TableSchema rea
.map(column -> column.getTypeInfo().getOdpsType())
.sorted(Enum::compareTo)
.collect(Collectors.toList());

/*
Comment it out and wait for the type inference to complete
if (!neededType.equals(realType)) {
log.error(
"odps_type_not_match, need_type="
Expand All @@ -158,6 +159,7 @@ public static boolean isSchemaMatch(OdpsTableInfo odpsTableInfo, TableSchema rea
+ JSON.toJSONString(realType));
return false;
}
*/

return true;
}
Expand Down
Loading