wip: Activate, test (and fix) on_disk caching for proc-macro expansions!

Also add manual test.

Author: futile

compiler/rustc_ast/src/tokenstream.rs

@@ -141,10 +141,13 @@ impl fmt::Debug for LazyAttrTokenStream {
 }
 
 impl<S: SpanEncoder> Encodable<S> for LazyAttrTokenStream {
-    fn encode(&self, s: &mut S) {
-        // TODO(pr-time): welp, do we really want this impl? maybe newtype wrapper?
-        // TODO(pr-time): (also) `.flattened()` here?
-        self.to_attr_token_stream().encode(s)
+    fn encode(&self, _s: &mut S) {
+        // TODO(pr-time): Just a reminder that this exists/was tried out,
+        // but probably not necessary anymore (see below).
+        // self.to_attr_token_stream().encode(s)
+        // We should not need to anymore, now that we `flatten`?
+        // Yep, that seems to be true! :)
+        panic!("Attempted to encode LazyAttrTokenStream");
     }
 }
 

compiler/rustc_expand/src/derive_macro_expansion.rs

@@ -2,6 +2,7 @@ use std::cell::Cell;
 use std::ptr;
 
 use rustc_ast::tokenstream::TokenStream;
+use rustc_data_structures::svh::Svh;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::LocalExpnId;
 use rustc_span::profiling::SpannedEventArgRecorder;
@@ -11,9 +12,9 @@ use crate::errors;
 
 pub(super) fn provide_derive_macro_expansion<'tcx>(
     tcx: TyCtxt<'tcx>,
-    key: (LocalExpnId, &'tcx TokenStream),
+    key: (LocalExpnId, Svh, &'tcx TokenStream),
 ) -> Result<&'tcx TokenStream, ()> {
-    let (invoc_id, input) = key;
+    let (invoc_id, _macro_crate_hash, input) = key;
 
     let res = with_context(|(ecx, client)| {
         let span = invoc_id.expn_data().call_site;

compiler/rustc_expand/src/proc_macro.rs

@@ -130,10 +130,18 @@ impl MultiItemModifier for DeriveProcMacro {
             let input = tcx.arena.alloc(input.flattened()) as &TokenStream;
             let invoc_id = ecx.current_expansion.id;
 
+            // TODO(pr-time): Just using the crate hash to notice when the proc-macro code has
+            // changed. How to *correctly* depend on exactly the macro definition?
+            // I.e., depending on the crate hash is just a HACK (and leaves garbage in the
+            // incremental compilation dir).
+            let macro_def_id = invoc_id.expn_data().macro_def_id.unwrap();
+            let proc_macro_crate_hash = tcx.crate_hash(macro_def_id.krate);
+
             assert_eq!(invoc_id.expn_data().call_site, span);
 
             let res = crate::derive_macro_expansion::enter_context((ecx, self.client), move || {
-                let res = tcx.derive_macro_expansion((invoc_id, input)).cloned();
+                let res =
+                    tcx.derive_macro_expansion((invoc_id, proc_macro_crate_hash, input)).cloned();
                 res
             });
 

compiler/rustc_middle/src/arena.rs

@@ -116,7 +116,7 @@ macro_rules! arena_types {
             [decode] specialization_graph: rustc_middle::traits::specialization_graph::Graph,
             [] crate_inherent_impls: rustc_middle::ty::CrateInherentImpls,
             [] hir_owner_nodes: rustc_hir::OwnerNodes<'tcx>,
-            [] token_stream: rustc_ast::tokenstream::TokenStream,
+            [decode] token_stream: rustc_ast::tokenstream::TokenStream,
         ]);
     )
 }

compiler/rustc_middle/src/query/keys.rs

@@ -3,6 +3,7 @@
 use std::ffi::OsStr;
 
 use rustc_ast::tokenstream::TokenStream;
+use rustc_data_structures::svh::Svh;
 use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId, LocalModDefId, ModDefId};
 use rustc_hir::hir_id::{HirId, OwnerId};
 use rustc_query_system::dep_graph::DepNodeIndex;
@@ -609,7 +610,7 @@ impl Key for (LocalDefId, HirId) {
     }
 }
 
-impl<'tcx> Key for (LocalExpnId, &'tcx TokenStream) {
+impl<'tcx> Key for (LocalExpnId, Svh, &'tcx TokenStream) {
     type Cache<V> = DefaultCache<Self, V>;
 
     fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {

compiler/rustc_middle/src/query/mod.rs

@@ -108,10 +108,11 @@ pub use plumbing::{IntoQueryParam, TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk};
 // Queries marked with `fatal_cycle` do not need the latter implementation,
 // as they will raise an fatal error on query cycles instead.
 rustc_queries! {
-    query derive_macro_expansion(key: (LocalExpnId, &'tcx TokenStream)) -> Result<&'tcx TokenStream, ()> {
-        eval_always
+    query derive_macro_expansion(key: (LocalExpnId, Svh, &'tcx TokenStream)) -> Result<&'tcx TokenStream, ()> {
+        // eval_always
         no_hash
         desc { "expanding a derive (proc) macro" }
+        cache_on_disk_if { true }
     }
 
     /// This exists purely for testing the interactions between delayed bugs and incremental.

compiler/rustc_middle/src/query/on_disk_cache.rs

@@ -773,6 +773,13 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>>
     }
 }
 
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx rustc_ast::tokenstream::TokenStream {
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
 macro_rules! impl_ref_decoder {
     (<$tcx:tt> $($ty:ty,)*) => {
         $(impl<'a, $tcx> Decodable<CacheDecoder<'a, $tcx>> for &$tcx [$ty] {

tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs

@@ -10,15 +10,11 @@ use proc_macro::TokenStream;
 pub fn derive(input: TokenStream) -> TokenStream {
     eprintln!("invoked");
 
-    r#"
+    return r#"
         pub mod nothing_mod {
-            // #[cfg(cfail1)]
             pub fn nothing() {
                 eprintln!("nothing");
             }
-
-            // #[cfg(cfail2)]
-            // fn nothingx() {}
         }
-    "#.parse().unwrap()
+    "#.parse().unwrap();
 }

tests/incremental/derive_macro_expansion/item_changed.rs

@@ -0,0 +1,37 @@
+// This test tests that derive-macro execution is cached.
+// HOWEVER, this test can currently only be checked manually,
+// by running it (through compiletest) with `-- --nocapture --verbose`.
+// The proc-macro (for `Nothing`) prints a message to stderr when invoked,
+// and this message should only be present during the second invocation
+// (which has `cfail2` set via cfg).
+// TODO(pr-time): Properly have the test check this, but how? UI-test that tests for `.stderr`?
+
+//@ aux-build:derive_nothing.rs
+//@ revisions:cfail1 cfail2
+//@ compile-flags: -Z query-dep-graph
+//@ build-pass
+
+#![feature(rustc_attrs)]
+#![feature(stmt_expr_attributes)]
+#![allow(dead_code)]
+#![crate_type = "rlib"]
+
+#![rustc_partition_codegened(module="proc_macro_unchanged-foo", cfg="cfail1")]
+#![rustc_partition_codegened(module="proc_macro_unchanged-foo", cfg="cfail2")]
+
+// `foo::nothing_mod` is created by the derive macro and doesn't change
+#![rustc_partition_reused(module="proc_macro_unchanged-foo", cfg="cfail2")]
+
+ #[macro_use]
+ extern crate derive_nothing;
+
+pub mod foo {
+    #[derive(Nothing)]
+    pub struct Foo;
+
+    pub fn use_foo(_f: Foo) {
+        nothing_mod::nothing();
+
+        eprintln!("foo used");
+    }
+}