Mirrors/nushell
2
0
Fork 0
mirror of https://github.com/nushell/nushell synced 2024-11-10 07:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity

Make pipeline metadata available to plugins (#13495)

Browse source 
# Description
Fixes an issue with pipeline metadata not being passed to plugins.
This commit is contained in:
Jack Wright 2024-08-02 11:01:20 -07:00 committed by GitHub
parent ca8eb856e8
commit d081e3386f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 208 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
crates/nu-plugin-core/src/interface/mod.rs
View file

@ -177,16 +177,19 @@ pub trait InterfaceManager {
) -> Result<PipelineData, ShellError> {
self.prepare_pipeline_data(match header {
PipelineDataHeader::Empty => PipelineData::Empty,
PipelineDataHeader::Value(value) => PipelineData::Value(value, None),
PipelineDataHeader::Value(value, metadata) => PipelineData::Value(value, metadata),
PipelineDataHeader::ListStream(info) => {
let handle = self.stream_manager().get_handle();
let reader = handle.read_stream(info.id, self.get_interface())?;
ListStream::new(reader, info.span, signals.clone()).into()
let ls = ListStream::new(reader, info.span, signals.clone());
PipelineData::ListStream(ls, info.metadata)
}
PipelineDataHeader::ByteStream(info) => {
let handle = self.stream_manager().get_handle();
let reader = handle.read_stream(info.id, self.get_interface())?;
ByteStream::from_result_iter(reader, info.span, signals.clone(), info.type_).into()
let bs =
ByteStream::from_result_iter(reader, info.span, signals.clone(), info.type_);
PipelineData::ByteStream(bs, info.metadata)
}
})
}
@ -248,26 +251,33 @@ pub trait Interface: Clone + Send {
Ok::<_, ShellError>((id, writer))
};
match self.prepare_pipeline_data(data, context)? {
PipelineData::Value(value, ..) => {
Ok((PipelineDataHeader::Value(value), PipelineDataWriter::None))
}
PipelineData::Value(value, metadata) => Ok((
PipelineDataHeader::Value(value, metadata),
PipelineDataWriter::None,
)),
PipelineData::Empty => Ok((PipelineDataHeader::Empty, PipelineDataWriter::None)),
PipelineData::ListStream(stream, ..) => {
PipelineData::ListStream(stream, metadata) => {
let (id, writer) = new_stream(LIST_STREAM_HIGH_PRESSURE)?;
Ok((
PipelineDataHeader::ListStream(ListStreamInfo {
id,
span: stream.span(),
metadata,
}),
PipelineDataWriter::ListStream(writer, stream),
))
}
PipelineData::ByteStream(stream, ..) => {
PipelineData::ByteStream(stream, metadata) => {
let span = stream.span();
let type_ = stream.type_();
if let Some(reader) = stream.reader() {
let (id, writer) = new_stream(RAW_STREAM_HIGH_PRESSURE)?;
let header = PipelineDataHeader::ByteStream(ByteStreamInfo { id, span, type_ });
let header = PipelineDataHeader::ByteStream(ByteStreamInfo {
id,
span,
type_,
metadata,
});
Ok((header, PipelineDataWriter::ByteStream(writer, reader)))
} else {
Ok((PipelineDataHeader::Empty, PipelineDataWriter::None))

33
crates/nu-plugin-core/src/interface/tests.rs
View file

@ -137,10 +137,16 @@ fn read_pipeline_data_empty() -> Result<(), ShellError> {
fn read_pipeline_data_value() -> Result<(), ShellError> {
let manager = TestInterfaceManager::new(&TestCase::new());
let value = Value::test_int(4);
let header = PipelineDataHeader::Value(value.clone());
let metadata = Some(PipelineMetadata {
data_source: DataSource::FilePath("/test/path".into()),
content_type: None,
});
let header = PipelineDataHeader::Value(value.clone(), metadata.clone());
match manager.read_pipeline_data(header, &Signals::empty())? {
PipelineData::Value(read_value, ..) => assert_eq!(value, read_value),
PipelineData::Value(read_value, read_metadata) => {
assert_eq!(value, read_value);
assert_eq!(metadata, read_metadata);
}
PipelineData::ListStream(..) => panic!("unexpected ListStream"),
PipelineData::ByteStream(..) => panic!("unexpected ByteStream"),
PipelineData::Empty => panic!("unexpected Empty"),
@ -161,9 +167,15 @@ fn read_pipeline_data_list_stream() -> Result<(), ShellError> {
}
test.add(StreamMessage::End(7));
let metadata = Some(PipelineMetadata {
data_source: DataSource::None,
content_type: Some("foobar".into()),
});
let header = PipelineDataHeader::ListStream(ListStreamInfo {
id: 7,
span: Span::test_data(),
metadata,
});
let pipe = manager.read_pipeline_data(header, &Signals::empty())?;
@ -204,10 +216,17 @@ fn read_pipeline_data_byte_stream() -> Result<(), ShellError> {
test.add(StreamMessage::End(12));
let test_span = Span::new(10, 13);
let metadata = Some(PipelineMetadata {
data_source: DataSource::None,
content_type: Some("foobar".into()),
});
let header = PipelineDataHeader::ByteStream(ByteStreamInfo {
id: 12,
span: test_span,
type_: ByteStreamType::Unknown,
metadata,
});
let pipe = manager.read_pipeline_data(header, &Signals::empty())?;
@ -251,9 +270,15 @@ fn read_pipeline_data_byte_stream() -> Result<(), ShellError> {
#[test]
fn read_pipeline_data_prepared_properly() -> Result<(), ShellError> {
let manager = TestInterfaceManager::new(&TestCase::new());
let metadata = Some(PipelineMetadata {
data_source: DataSource::None,
content_type: Some("foobar".into()),
});
let header = PipelineDataHeader::ListStream(ListStreamInfo {
id: 0,
span: Span::test_data(),
metadata,
});
match manager.read_pipeline_data(header, &Signals::empty())? {
PipelineData::ListStream(_, meta) => match meta {
@ -301,7 +326,7 @@ fn write_pipeline_data_value() -> Result<(), ShellError> {
interface.init_write_pipeline_data(PipelineData::Value(value.clone(), None), &())?;
match header {
PipelineDataHeader::Value(read_value) => assert_eq!(value, read_value),
PipelineDataHeader::Value(read_value, _) => assert_eq!(value, read_value),
_ => panic!("unexpected header: {header:?}"),
}

18
crates/nu-plugin-core/src/serializers/tests.rs
View file

@ -6,7 +6,8 @@ macro_rules! generate_tests {
StreamData,
};
use nu_protocol::{
LabeledError, PluginSignature, Signature, Span, Spanned, SyntaxShape, Value,
DataSource, LabeledError, PipelineMetadata, PluginSignature, Signature, Span, Spanned,
SyntaxShape, Value,
};
#[test]
@ -123,10 +124,15 @@ macro_rules! generate_tests {
)],
};
let metadata = Some(PipelineMetadata {
data_source: DataSource::None,
content_type: Some("foobar".into()),
});
let plugin_call = PluginCall::Run(CallInfo {
name: name.clone(),
call: call.clone(),
input: PipelineDataHeader::Value(input.clone()),
input: PipelineDataHeader::Value(input.clone(), metadata.clone()),
});
let plugin_input = PluginInput::Call(1, plugin_call);
@ -144,7 +150,7 @@ macro_rules! generate_tests {
match returned {
PluginInput::Call(1, PluginCall::Run(call_info)) => {
assert_eq!(name, call_info.name);
assert_eq!(PipelineDataHeader::Value(input), call_info.input);
assert_eq!(PipelineDataHeader::Value(input, metadata), call_info.input);
assert_eq!(call.head, call_info.call.head);
assert_eq!(call.positional.len(), call_info.call.positional.len());
@ -305,7 +311,7 @@ macro_rules! generate_tests {
match returned {
PluginOutput::CallResponse(
4,
PluginCallResponse::PipelineData(PipelineDataHeader::Value(returned_value)),
PluginCallResponse::PipelineData(PipelineDataHeader::Value(returned_value, _)),
) => {
assert_eq!(value, returned_value)
}
@ -325,7 +331,7 @@ macro_rules! generate_tests {
span,
);
let response = PluginCallResponse::PipelineData(PipelineDataHeader::Value(value));
let response = PluginCallResponse::PipelineData(PipelineDataHeader::value(value));
let output = PluginOutput::CallResponse(5, response);
let encoder = $encoder;
@ -341,7 +347,7 @@ macro_rules! generate_tests {
match returned {
PluginOutput::CallResponse(
5,
PluginCallResponse::PipelineData(PipelineDataHeader::Value(returned_value)),
PluginCallResponse::PipelineData(PipelineDataHeader::Value(returned_value, _)),
) => {
assert_eq!(span, returned_value.span());

68
crates/nu-plugin-engine/src/interface/tests.rs
View file

@ -17,8 +17,9 @@ use nu_plugin_protocol::{
use nu_protocol::{
ast::{Math, Operator},
engine::Closure,
ByteStreamType, CustomValue, IntoInterruptiblePipelineData, IntoSpanned, PipelineData,
PluginMetadata, PluginSignature, ShellError, Signals, Span, Spanned, Value,
ByteStreamType, CustomValue, DataSource, IntoInterruptiblePipelineData, IntoSpanned,
PipelineData, PipelineMetadata, PluginMetadata, PluginSignature, ShellError, Signals, Span,
Spanned, Value,
};
use serde::{Deserialize, Serialize};
use std::{
@ -52,10 +53,7 @@ fn manager_consume_all_exits_after_streams_and_interfaces_are_dropped() -> Resul
// Create a stream...
let stream = manager.read_pipeline_data(
PipelineDataHeader::ListStream(ListStreamInfo {
id: 0,
span: Span::test_data(),
}),
PipelineDataHeader::list_stream(ListStreamInfo::new(0, Span::test_data())),
&Signals::empty(),
)?;
@ -108,10 +106,7 @@ fn manager_consume_all_propagates_io_error_to_readers() -> Result<(), ShellError
test.set_read_error(test_io_error());
let stream = manager.read_pipeline_data(
PipelineDataHeader::ListStream(ListStreamInfo {
id: 0,
span: Span::test_data(),
}),
PipelineDataHeader::list_stream(ListStreamInfo::new(0, Span::test_data())),
&Signals::empty(),
)?;
@ -154,11 +149,11 @@ fn manager_consume_all_propagates_message_error_to_readers() -> Result<(), Shell
test.add(invalid_output());
let stream = manager.read_pipeline_data(
PipelineDataHeader::ByteStream(ByteStreamInfo {
id: 0,
span: Span::test_data(),
type_: ByteStreamType::Unknown,
}),
PipelineDataHeader::byte_stream(ByteStreamInfo::new(
0,
Span::test_data(),
ByteStreamType::Unknown,
)),
&Signals::empty(),
)?;
@ -331,10 +326,10 @@ fn manager_consume_call_response_forwards_to_subscriber_with_pipeline_data(
manager.consume(PluginOutput::CallResponse(
0,
PluginCallResponse::PipelineData(PipelineDataHeader::ListStream(ListStreamInfo {
id: 0,
span: Span::test_data(),
})),
PluginCallResponse::PipelineData(PipelineDataHeader::list_stream(ListStreamInfo::new(
0,
Span::test_data(),
))),
))?;
for i in 0..2 {
@ -375,18 +370,18 @@ fn manager_consume_call_response_registers_streams() -> Result<(), ShellError> {
// Check list streams, byte streams
manager.consume(PluginOutput::CallResponse(
0,
PluginCallResponse::PipelineData(PipelineDataHeader::ListStream(ListStreamInfo {
id: 0,
span: Span::test_data(),
})),
PluginCallResponse::PipelineData(PipelineDataHeader::list_stream(ListStreamInfo::new(
0,
Span::test_data(),
))),
))?;
manager.consume(PluginOutput::CallResponse(
1,
PluginCallResponse::PipelineData(PipelineDataHeader::ByteStream(ByteStreamInfo {
id: 1,
span: Span::test_data(),
type_: ByteStreamType::Unknown,
})),
PluginCallResponse::PipelineData(PipelineDataHeader::byte_stream(ByteStreamInfo::new(
1,
Span::test_data(),
ByteStreamType::Unknown,
))),
))?;
// ListStream should have one
@ -442,10 +437,7 @@ fn manager_consume_engine_call_forwards_to_subscriber_with_pipeline_data() -> Re
span: Span::test_data(),
},
positional: vec![],
input: PipelineDataHeader::ListStream(ListStreamInfo {
id: 2,
span: Span::test_data(),
}),
input: PipelineDataHeader::list_stream(ListStreamInfo::new(2, Span::test_data())),
redirect_stdout: false,
redirect_stderr: false,
},
@ -806,6 +798,11 @@ fn interface_write_plugin_call_writes_run_with_value_input() -> Result<(), Shell
let manager = test.plugin("test");
let interface = manager.get_interface();
let metadata0 = PipelineMetadata {
data_source: DataSource::None,
content_type: Some("baz".into()),
};
let result = interface.write_plugin_call(
PluginCall::Run(CallInfo {
name: "foo".into(),
@ -814,7 +811,7 @@ fn interface_write_plugin_call_writes_run_with_value_input() -> Result<(), Shell
positional: vec![],
named: vec![],
},
input: PipelineData::Value(Value::test_int(-1), None),
input: PipelineData::Value(Value::test_int(-1), Some(metadata0.clone())),
}),
None,
)?;
@ -826,7 +823,10 @@ fn interface_write_plugin_call_writes_run_with_value_input() -> Result<(), Shell
PluginCall::Run(CallInfo { name, input, .. }) => {
assert_eq!("foo", name);
match input {
PipelineDataHeader::Value(value) => assert_eq!(-1, value.as_int()?),
PipelineDataHeader::Value(value, metadata) => {
assert_eq!(-1, value.as_int()?);
assert_eq!(metadata0, metadata.expect("there should be metadata"));
}
_ => panic!("unexpected input header: {input:?}"),
}
}

45
crates/nu-plugin-protocol/src/lib.rs
View file

@ -23,7 +23,7 @@ pub mod test_util;
use nu_protocol::{
ast::Operator, engine::Closure, ByteStreamType, Config, DeclId, LabeledError, PipelineData,
PluginMetadata, PluginSignature, ShellError, Span, Spanned, Value,
PipelineMetadata, PluginMetadata, PluginSignature, ShellError, Span, Spanned, Value,
};
use nu_utils::SharedCow;
use serde::{Deserialize, Serialize};
@ -78,7 +78,7 @@ pub enum PipelineDataHeader {
/// No input
Empty,
/// A single value
Value(Value),
Value(Value, Option<PipelineMetadata>),
/// Initiate [`nu_protocol::PipelineData::ListStream`].
///
/// Items are sent via [`StreamData`]
@ -94,11 +94,23 @@ impl PipelineDataHeader {
pub fn stream_id(&self) -> Option<StreamId> {
match self {
PipelineDataHeader::Empty => None,
PipelineDataHeader::Value(_) => None,
PipelineDataHeader::Value(_, _) => None,
PipelineDataHeader::ListStream(info) => Some(info.id),
PipelineDataHeader::ByteStream(info) => Some(info.id),
}
}
pub fn value(value: Value) -> Self {
PipelineDataHeader::Value(value, None)
}
pub fn list_stream(info: ListStreamInfo) -> Self {
PipelineDataHeader::ListStream(info)
}
pub fn byte_stream(info: ByteStreamInfo) -> Self {
PipelineDataHeader::ByteStream(info)
}
}
/// Additional information about list (value) streams
@ -106,6 +118,18 @@ impl PipelineDataHeader {
pub struct ListStreamInfo {
pub id: StreamId,
pub span: Span,
pub metadata: Option<PipelineMetadata>,
}
impl ListStreamInfo {
/// Create a new `ListStreamInfo` with a unique ID
pub fn new(id: StreamId, span: Span) -> Self {
ListStreamInfo {
id,
span,
metadata: None,
}
}
}
/// Additional information about byte streams
@ -115,6 +139,19 @@ pub struct ByteStreamInfo {
pub span: Span,
#[serde(rename = "type")]
pub type_: ByteStreamType,
pub metadata: Option<PipelineMetadata>,
}
impl ByteStreamInfo {
/// Create a new `ByteStreamInfo` with a unique ID
pub fn new(id: StreamId, span: Span, type_: ByteStreamType) -> Self {
ByteStreamInfo {
id,
span,
type_,
metadata: None,
}
}
}
/// Calls that a plugin can execute. The type parameter determines the input type.
@ -344,7 +381,7 @@ impl PluginCallResponse<PipelineDataHeader> {
if value.is_nothing() {
PluginCallResponse::PipelineData(PipelineDataHeader::Empty)
} else {
PluginCallResponse::PipelineData(PipelineDataHeader::Value(value))
PluginCallResponse::PipelineData(PipelineDataHeader::value(value))
}
}
}

35
crates/nu-plugin/src/plugin/interface/tests.rs
View file

@ -55,10 +55,7 @@ fn manager_consume_all_exits_after_streams_and_interfaces_are_dropped() -> Resul
// Create a stream...
let stream = manager.read_pipeline_data(
PipelineDataHeader::ListStream(ListStreamInfo {
id: 0,
span: Span::test_data(),
}),
PipelineDataHeader::list_stream(ListStreamInfo::new(0, Span::test_data())),
&Signals::empty(),
)?;
@ -111,10 +108,7 @@ fn manager_consume_all_propagates_io_error_to_readers() -> Result<(), ShellError
test.set_read_error(test_io_error());
let stream = manager.read_pipeline_data(
PipelineDataHeader::ListStream(ListStreamInfo {
id: 0,
span: Span::test_data(),
}),
PipelineDataHeader::list_stream(ListStreamInfo::new(0, Span::test_data())),
&Signals::empty(),
)?;
@ -157,11 +151,11 @@ fn manager_consume_all_propagates_message_error_to_readers() -> Result<(), Shell
test.add(invalid_input());
let stream = manager.read_pipeline_data(
PipelineDataHeader::ByteStream(ByteStreamInfo {
id: 0,
span: Span::test_data(),
type_: ByteStreamType::Unknown,
}),
PipelineDataHeader::byte_stream(ByteStreamInfo::new(
0,
Span::test_data(),
ByteStreamType::Unknown,
)),
&Signals::empty(),
)?;
@ -414,10 +408,7 @@ fn manager_consume_call_run_forwards_to_receiver_with_pipeline_data() -> Result<
positional: vec![],
named: vec![],
},
input: PipelineDataHeader::ListStream(ListStreamInfo {
id: 6,
span: Span::test_data(),
}),
input: PipelineDataHeader::list_stream(ListStreamInfo::new(6, Span::test_data())),
}),
))?;
@ -556,10 +547,10 @@ fn manager_consume_engine_call_response_forwards_to_subscriber_with_pipeline_dat
manager.consume(PluginInput::EngineCallResponse(
0,
EngineCallResponse::PipelineData(PipelineDataHeader::ListStream(ListStreamInfo {
id: 0,
span: Span::test_data(),
})),
EngineCallResponse::PipelineData(PipelineDataHeader::list_stream(ListStreamInfo::new(
0,
Span::test_data(),
))),
))?;
for i in 0..2 {
@ -707,7 +698,7 @@ fn interface_write_response_with_value() -> Result<(), ShellError> {
assert_eq!(33, id, "id");
match response {
PluginCallResponse::PipelineData(header) => match header {
PipelineDataHeader::Value(value) => assert_eq!(6, value.as_int()?),
PipelineDataHeader::Value(value, _) => assert_eq!(6, value.as_int()?),
_ => panic!("unexpected pipeline data header: {header:?}"),
},
_ => panic!("unexpected response: {response:?}"),

6
crates/nu-protocol/src/pipeline/metadata.rs
View file

@ -1,7 +1,9 @@
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
/// Metadata that is valid for the whole [`PipelineData`](crate::PipelineData)
#[derive(Debug, Default, Clone)]
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
pub struct PipelineMetadata {
pub data_source: DataSource,
pub content_type: Option<String>,
@ -27,7 +29,7 @@ impl PipelineMetadata {
///
/// This can either be a particular family of commands (useful so downstream commands can adjust
/// the presentation e.g. `Ls`) or the opened file to protect against overwrite-attempts properly.
#[derive(Debug, Default, Clone)]
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
pub enum DataSource {
Ls,
HtmlThemes,

8
crates/nu_plugin_nu_example/nu_plugin_nu_example.nu
View file

@ -7,7 +7,7 @@
# language without adding any extra dependencies to our tests.
const NUSHELL_VERSION = "0.96.2"
const PLUGIN_VERSION = "0.1.0" # bump if you change commands!
const PLUGIN_VERSION = "0.1.1" # bump if you change commands!
def main [--stdio] {
if ($stdio) {
@ -133,7 +133,7 @@ def process_call [
# Create a Value of type List that will be encoded and sent to Nushell
let value = {
Value: {
Value: [{
List: {
vals: (0..9 | each { |x|
{
@ -157,7 +157,7 @@ def process_call [
}),
span: $span
}
}
}, null]
}
write_response $id { PipelineData: $value }
@ -265,4 +265,4 @@ def start_plugin [] {
}) |
each { from json | handle_input } |
ignore
}
}

95
crates/nu_plugin_python/nu_plugin_python_example.py
View file

@ -28,7 +28,7 @@ import json
NUSHELL_VERSION = "0.96.2"
PLUGIN_VERSION = "0.1.0" # bump if you change commands!
PLUGIN_VERSION = "0.1.1" # bump if you change commands!
def signatures():
@ -125,31 +125,31 @@ def process_call(id, plugin_call):
span = plugin_call["call"]["head"]
# Creates a Value of type List that will be encoded and sent to Nushell
def f(x, y): return {
"Int": {
"val": x * y,
"span": span
}
}
def f(x, y):
return {"Int": {"val": x * y, "span": span}}
value = {
"Value": {
"List": {
"vals": [
{
"Record": {
"val": {
"one": f(x, 0),
"two": f(x, 1),
"three": f(x, 2),
},
"span": span
"Value": [
{
"List": {
"vals": [
{
"Record": {
"val": {
"one": f(x, 0),
"two": f(x, 1),
"three": f(x, 2),
},
"span": span,
}
}
} for x in range(0, 10)
],
"span": span
}
}
for x in range(0, 10)
],
"span": span,
}
},
None,
]
}
write_response(id, {"PipelineData": value})
@ -172,7 +172,7 @@ def tell_nushell_hello():
"Hello": {
"protocol": "nu-plugin", # always this value
"version": NUSHELL_VERSION,
"features": []
"features": [],
}
}
sys.stdout.write(json.dumps(hello))
@ -200,22 +200,26 @@ def write_error(id, text, span=None):
Use this error format to send errors to nushell in response to a plugin call. The ID of the
plugin call is required.
"""
error = {
"Error": {
"msg": "ERROR from plugin",
"labels": [
{
"text": text,
"span": span,
}
],
error = (
{
"Error": {
"msg": "ERROR from plugin",
"labels": [
{
"text": text,
"span": span,
}
],
}
}
} if span is not None else {
"Error": {
"msg": "ERROR from plugin",
"help": text,
if span is not None
else {
"Error": {
"msg": "ERROR from plugin",
"help": text,
}
}
}
)
write_response(id, error)
@ -230,11 +234,14 @@ def handle_input(input):
elif "Call" in input:
[id, plugin_call] = input["Call"]
if plugin_call == "Metadata":
write_response(id, {
"Metadata": {
"version": PLUGIN_VERSION,
}
})
write_response(
id,
{
"Metadata": {
"version": PLUGIN_VERSION,
}
},
)
elif plugin_call == "Signature":
write_response(id, signatures())
elif "Run" in plugin_call:
@ -258,4 +265,4 @@ if __name__ == "__main__":
if len(sys.argv) == 2 and sys.argv[1] == "--stdio":
plugin()
else:
print("Run me from inside nushell!")
print("Run me from inside nushell!")

2
crates/nu_plugin_stress_internals/src/main.rs
View file

@ -178,7 +178,7 @@ fn handle_message(
id,
{
"PipelineData": {
"Value": return_value
"Value": [return_value, null]
}
}
]