尝试过把sql逻辑拆开成多个view, 但是sql逻辑优化的时候还是会合并到一起,无法绕过
Caused by: org.codehaus.janino.InternalCompilerException: Code of method "map(Ljava/lang/Object;)Ljava/lang/Object;" of class "ExpressionReducer$3674" grows beyond 64 KB at org.codehaus.janino.CodeContext.makeSpace(CodeContext.java:1009) at org.codehaus.janino.CodeContext.write(CodeContext.java:901) at org.codehaus.janino.UnitCompiler.writeShort(UnitCompiler.java:12195) at org.codehaus.janino.UnitCompiler.pushConstant(UnitCompiler.java:10660) at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5553) at org.codehaus.janino.UnitCompiler.access$9300(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$16.visitIntegerLiteral(UnitCompiler.java:4423) at org.codehaus.janino.UnitCompiler$16.visitIntegerLiteral(UnitCompiler.java:4394) at org.codehaus.janino.Java$IntegerLiteral.accept(Java.java:5442) at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4394) at org.codehaus.janino.UnitCompiler.fakeCompile(UnitCompiler.java:3719) at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5569) at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5165) at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4421) at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4394) at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5062) at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4394) at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5575) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3781) at org.codehaus.janino.UnitCompiler.access$5900(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3760) at org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3732) at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5062) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3732) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2360) at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1494) at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1487) at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2871) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1553) at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1493) at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1487) at org.codehaus.janino.Java$Block.accept(Java.java:2776) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2465) at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1495) at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1487) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2947) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388) at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357) at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432) at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411) at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406) at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406) at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378) ... 51 more*来自志愿者整理的flink邮件归档
这个限制不仅仅在janino这边,我们这边已经尝试去除了janino这边的64kb的限制,但是逻辑运行到jdk这边也出现了同样的限制。目前spark是通过spark.conf.set("spark.sql.codegen.wholeStage",false)来避免类似的问题,所以我觉得最好的方法是可以通过配置来限制一些SQL逻辑计划的优化,避免不同的view的逻辑合并到一起,这样就能通过将拆分sql来将逻辑拆分到不同的function中,从而避免一个类函数太大。*来自志愿者整理的flink邮件归档
版权声明:本文内容由阿里云实名注册用户自发贡献,版权归原作者所有,阿里云开发者社区不拥有其著作权,亦不承担相应法律责任。具体规则请查看《阿里云开发者社区用户服务协议》和《阿里云开发者社区知识产权保护指引》。如果您发现本社区中有涉嫌抄袭的内容,填写侵权投诉表单进行举报,一经查实,本社区将立刻删除涉嫌侵权内容。