From 827029417b88fd0b1a0fefec029adb0ad4579546 Mon Sep 17 00:00:00 2001 From: Kristian Aune Date: Fri, 25 Aug 2023 12:59:20 +0200 Subject: [PATCH] Execute the notebook --- .../examples/multi-vector-indexing.ipynb | 298 ++++++++++++++++-- 1 file changed, 275 insertions(+), 23 deletions(-) diff --git a/docs/sphinx/source/examples/multi-vector-indexing.ipynb b/docs/sphinx/source/examples/multi-vector-indexing.ipynb index 88a38f39..7f7c0d88 100644 --- a/docs/sphinx/source/examples/multi-vector-indexing.ipynb +++ b/docs/sphinx/source/examples/multi-vector-indexing.ipynb @@ -21,14 +21,15 @@ "* File export, model files in the application package, deployment from files.\n", "* How to control text search result highlighting.\n", "\n", - "This notebook requires pyvespa >= 0.37.0.\n", + "This notebook requires pyvespa >= 0.37.1\n", + "and the [Vespa CLI](https://pyvespa.readthedocs.io/en/latest/reads-writes.html#Feed-using-Vespa-CLI).\n", "\n", "## Create the application" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "4d7baafb", "metadata": {}, "outputs": [], @@ -119,12 +120,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "32b87bdd", "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " % Total % Received % Xferd Average Speed Time Time Time Current\n", + " Dload Upload Total Spent Left Speed\n", + "100 694k 100 694k 0 0 721k 0 --:--:-- --:--:-- --:--:-- 725k\n", + " % Total % Received % Xferd Average Speed Time Time Time Current\n", + " Dload Upload Total Spent Left Speed\n", + " 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\n", + "100 32.3M 100 32.3M 0 0 7766k 0 0:00:04 0:00:04 --:--:-- 10.1M\n" + ] + } + ], "source": [ "! mkdir -p pkg/model\n", "! curl -L -o pkg/model/tokenizer.json \\\n", @@ -144,10 +159,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "02821efd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for configuration server, 0/300 seconds...\n", + "Waiting for configuration server, 5/300 seconds...\n", + "Waiting for application status, 0/300 seconds...\n", + "Waiting for application status, 5/300 seconds...\n", + "Waiting for application status, 10/300 seconds...\n", + "Waiting for application status, 15/300 seconds...\n", + "Waiting for application status, 20/300 seconds...\n", + "Waiting for application status, 25/300 seconds...\n", + "Finished deployment.\n" + ] + } + ], "source": [ "from vespa.deployment import VespaDocker\n", "\n", @@ -167,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "1a74eff0", "metadata": {}, "outputs": [], @@ -187,10 +218,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "8130fdfa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"feeder.seconds\": 2.650,\r\n", + " \"feeder.ok.count\": 8,\r\n", + " \"feeder.ok.rate\": 3.019,\r\n", + " \"feeder.error.count\": 0,\r\n", + " \"feeder.inflight.count\": 0,\r\n", + " \"http.request.count\": 8,\r\n", + " \"http.request.bytes\": 12958,\r\n", + " \"http.request.MBps\": 0.005,\r\n", + " \"http.exception.count\": 0,\r\n", + " \"http.response.count\": 8,\r\n", + " \"http.response.bytes\": 674,\r\n", + " \"http.response.MBps\": 0.000,\r\n", + " \"http.response.error.count\": 0,\r\n", + " \"http.response.latency.millis.min\": 1159,\r\n", + " \"http.response.latency.millis.avg\": 1445,\r\n", + " \"http.response.latency.millis.max\": 2647,\r\n", + " \"http.response.code.counts\": {\r\n", + " \"200\": 8\r\n", + " }\r\n", + "}\r\n" + ] + } + ], "source": [ "! vespa config set target local\n", "! vespa feed articles.jsonl" @@ -206,14 +265,46 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "06dd6e2a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'id:wikipedia:wiki::8496',\n", + " 'relevance': 0.0,\n", + " 'source': 'wiki_content',\n", + " 'fields': {'sddocname': 'wiki',\n", + " 'paragraphs': ['The way the word is used varies. Most societies have rites of passage to mark the change from childhood to adulthood. These ceremonies may be quite elaborate. During puberty, rapid mental and physical development occurs. Adolescence is the name for this transition period from childhood to adulthood.',\n", + " '\"Teenager\" is mainly an English word, as many foreign languages do not include a suffix in their translations of the numbers 13 to 19. In non-English speaking countries, people between these ages may be called adolescents, youths, young adults, or just children, depending on the culture.',\n", + " 'The life of a teenager seems to change daily. Constantly exposed to new ideas, social situations and people, teenagers work to develop their personalities and interests during this time of great change. Before their teenage years, these adolescents focused on school, play, and gaining approval from their parents.'],\n", + " 'documentid': 'id:wikipedia:wiki::8496',\n", + " 'title': 'Teenager',\n", + " 'url': 'https://simple.wikipedia.org/wiki?curid=8496'}},\n", + " {'id': 'id:wikipedia:wiki::797944',\n", + " 'relevance': 0.0,\n", + " 'source': 'wiki_content',\n", + " 'fields': {'sddocname': 'wiki',\n", + " 'paragraphs': ['Abella Danger made her pornography debut in July 2014 for Bang Bros. She has appeared in about 1010 credited scenes. She has appeared in mainstream news media other than adult news media, including the websites \"Elite Daily\" and \"International Business Times\".',\n", + " 'In 2018, \"Fortune\" said she was one of the most popular and in-demand performers in the pornographic business.',\n", + " \"Amongst many awards, she won Best Pornographic actor in 2021 (Pornovizija '21), nominated by the competent jury committee.\",\n", + " 'Abella belongs to a Jewish-Ukrainian family. She started as a ballet dancer when she was only three years old.'],\n", + " 'documentid': 'id:wikipedia:wiki::797944',\n", + " 'title': 'Abella Danger',\n", + " 'url': 'https://simple.wikipedia.org/wiki?curid=797944'}}]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "result = app.query(body={\n", " 'yql': 'select * from wiki where true',\n", - " 'ranking.profile': 'unranked'\n", + " 'ranking.profile': 'unranked',\n", + " 'hits': 2\n", "})\n", "result.hits" ] @@ -228,15 +319,81 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "0138aa1d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'id:wikipedia:wiki::9985',\n", + " 'relevance': 4.88768243450246,\n", + " 'source': 'wiki_content',\n", + " 'fields': {'sddocname': 'wiki',\n", + " 'paragraphs': ['The 24-hour clock is a way of telling the time in which the day runs from midnight to midnight and is divided into 24 hours, numbered from 0 to 23. It does not use a.m. or p.m. This system is also referred to (only in the US and the English speaking parts of Canada) as military time or (only in the United Kingdom and now very rarely) as continental time. In some parts of the world, it is called railway time. Also, the international standard notation of time (ISO 8601) is based on this format.',\n", + " 'A time in the 24-hour clock is written in the form hours:minutes (for example, 01:23), or hours:minutes:seconds (01:23:45). Numbers under 10 have a zero in front (called a leading zero); e.g. 09:07. Under the 24-hour clock system, the day begins at midnight, 00:00, and the last minute of the day begins at 23:59 and ends at 24:00, which is identical to 00:00 of the following day. 12:00 can only be mid-day. Midnight is called 24:00 and is used to mean the end of the day and 00:00 is used to mean the beginning of the day. For example, you would say \"Tuesday at 24:00\" and \"Wednesday at 00:00\" to mean exactly the same time.',\n", + " 'However, the US military prefers not to say 24:00 - they do not like to have two names for the same thing, so they always say \"23:59\", which is one minute before midnight.',\n", + " '24-hour clock time is used in computers, military, public safety, and transport. In many Asian, European and Latin American countries people use it to write the time. Many European people use it in speaking.',\n", + " 'In railway timetables 24:00 means the \"end\" of the day. For example, a train due to arrive at a station during the last minute of a day arrives at 24:00; but trains which depart during the first minute of the day go at 00:00.'],\n", + " 'documentid': 'id:wikipedia:wiki::9985',\n", + " 'title': '24-hour clock',\n", + " 'url': 'https://simple.wikipedia.org/wiki?curid=9985'}},\n", + " {'id': 'id:wikipedia:wiki::65655',\n", + " 'relevance': 0.8262847635070322,\n", + " 'source': 'wiki_content',\n", + " 'fields': {'sddocname': 'wiki',\n", + " 'paragraphs': [\"Ronaldo began his professional career with Sporting CP at age 17 in 2002, and signed for Manchester United a year later. He won the FA Cup in his first season, and then won three back-to-back Premier League titles: in 2006-07, 2007-08, and 2008-09. In 2007-08, Ronaldo, helped United win the UEFA Champions League. In 2008-09, he won his first FIFA Club World Cup in December 2008, and he also won his first Ballon d'Or. At one point Ronaldo was the most expensive professional footballer of all time, after moving from Manchester United to Real Madrid for approximately £80 m in July 2009.\",\n", + " \"He won his first trophy with Real Madrid in 2011, the 2010-11 Copa del Rey. In the next season, he won his first La Liga title with the club, the 2011-12 La Liga. In the 2012-13 season he won the Supercopa de España. In the next season, the 2013-14 season, he won his second Ballon d'Or. Then he won the Copa del Rey, and he also won his second Champions League with a record 17 goals. The following year, Ronaldo won the Ballon d'Or again, along with his second FIFA Club World Cup in December 2014. In 2016, Ronaldo won his third Champions League, and scored the winning penalty in the final against Atlético Madrid. He won his fourth Ballon d'or the next season, his second La Liga title for the first time in five years, another Champions League, and his second Club World Cup. Ronaldo's last season with Real Madrid was the 2017-18 season, where he won his fifth Ballon d'Or in 2017, and also won his fifth Champions League and scored two goals in the final against Juventus. With his third consecutive Champions League, he became the first player to win the UEFA Champions League five times. He would later go on to transfer to Juventus in July 2018. Ronaldo left the club by holding the record for being the top goal scorer in Real Madrid's history, and remaining as the only player in La Liga's history to score 30 or more goals in six consecutive seasons.\",\n", + " \"Ronaldo began his career with Portugal at age 18. He scored his first goal at UEFA Euro 2004 and helped Portugal reach the final, although they lost to Greece 1-0. The first World Cup he played at was the 2006 FIFA World Cup. He scored a goal and helped Portugal earn fourth place. Two years later, he became Portugal's full captain. Since then, he has appeared at three Euro's: 2008, 2012, and 2016, and two World Cups: 2014 and 2018. He helped Portugal win their first major international trophy at Euro 2016, and their second three years later with the UEFA Nations League.\",\n", + " 'He currently Plays for Manchester United and his shirt number is 7. He is now considered the greatest of all time with the most goals of 800+ in his all time of carrer.',\n", + " 'Cristiano Ronaldo dos Santos Aveiro was born in Funchal, Madeira Islands to Maria Dolores dos Santos and José Dinis Aveiro. He has one brother named Hugo, and two sisters named Katia and Elma. Ronaldo was diagnosed with a racing heart (Tachycardia) at age 15 In 1997, 12-year-old Ronaldo went on a trial with Sporting CP. He impressed the club enough to be signed for £1,500. He then moved from Funchal to Lisbon, to join the Sporting youth academy.',\n", + " 'Ronaldo played his first professional match against Inter Milan on 14 August 2002. He came in as a sub during the second half. On 7 October 2002, Ronaldo played a role in his first game in the Portuguese Primeira Liga, against Moreirense. He scored two goals and Sporting won 3–0. Ronaldo came to the attention of Manchester United manager Alex Ferguson in August 2003, when Sporting defeated United 3–1 in the first game ever played at the Estádio José Alvalade in Lisbon. His performance impressed the United players, who told Ferguson to sign him.',\n", + " \"On 12 August 2003, Ronaldo joined Manchester United from Sporting CP for a fee of £12.24 million. He was Manchester United's first Portuguese player. He wanted the number 28, the number he wore at Sporting, but was eventually given the number 7. This number had been worn by George Best, Eric Cantona and David Beckham before him. He played his first game for the club on 16 August 2003 in a 4-0 win against Bolton Wanderers. Many people were impressed with his debut, including legendary United player George Best. Ronaldo's first goal with Manchester United was a free kick. He scored it in a 3-0 win against Portsmouth on 1 November 2003.Ronaldo won his first trophy in England, the 2003-04 FA Cup, in May 2004, when Manchester United beat Millwall 3-0. Ronaldo scored the first goal of the match.\",\n", + " \"Ronaldo scored his first UEFA Champions League goals in a 7-1 victory against A.S. Roma in April 2007. He scored two goals that day in what was his 30th Champions League match. He played the FA Cup Final that year, but United lost 1-0 to Chelsea. Even though he lost the FA Cup final, he didn't end the season without a title, because he won his first Premier League title.\",\n", + " 'In the 2007-08 season, Ronaldo scored his first and only hat-trick for Manchester United in a 6–0 win against Newcastle United on 12 January 2008. In the 2008 Champions League final against Chelsea, he scored a header as the match ended 1-1 after extra time. Although he missed his penalty, Manchester United won the shootout 6-5 and Ronaldo won his first UEFA Champions League. On 15 November 2008, Ronaldo scored his 100th goal for United in a 5-0 win against Stoke City. He also scored two free-kicks: the first one was his 100th goal. He scored a total of 42 goals and won the European Golden Boot, an award given to the top scorer of every European national league.',\n", + " 'He won the first FIFA Puskas Award in 2009. The Puskas Award is given to whoever scores the best goal of that year. The goal was a 40-yard strike into the top-left corner against FC Porto on 15 April 2009 in the Champions League quarter finals. That goal was the only goal of the game. It was also an important goal because it sent United to the semi-finals. In the semi-final against Arsenal, Ronaldo scored two goals. One of them was a free kick from 40 yards out. His goals helped Man United qualify to the final, where they lost to Barcelona 2-0.',\n", + " \"Ronaldo joined Real Madrid on 1 July 2009 for a fee of €94 million. That was a world record transfer fee at the time. He also signed a six-year contract with the club. At his presentation as a Real Madrid player, 80,000 people greeted him at the Santiago Bernabéu Stadium. This is the world record, breaking the 25-year record of 75,000 people at Diego Maradona's presentation for Napoli. He wore the number 9 in his first season because number 7 was taken by Raúl González. Ronaldo had to wait until Raul left the club in the summer of 2010 to wear number 7. He made his debut (first appearance) on 29 August 2009, in a La Liga game against Deportivo de La Coruña. He scored a goal and Real Madrid won 3-2. On 23 October 2010, Ronaldo scored 4 goals in a match for the first time in his career during a 6-1 win against Racing de Santander. Ronaldo scored a header in extra time in the 2011 Copa del Rey final against rivals Barcelona. His goal was the match winner, so that was Ronaldo's first trophy in Spain. At the end of the 2010-11 season, he became the first player to score 40 goals in La Liga.\",\n", + " 'On 2 November 2011, Ronaldo scored both goals in a 2-0 Champions League group stage win against Olympique Lyon. The second goal was his 100th goal for Real Madrid. He achieved this in just 105 matches. He scored his 100th La Liga goal for his club in just 92 appearances in a 5-1 win against Real Sociedad on 24 March 2012.',\n", + " \"Ronaldo began the 2012-13 season by winning the 2012 Supercopa de España. He scored in both legs, as Real Madrid won against Barca. In October 2012, he scored his first hat-trick in the Champions League in a 4-1 win against Ajax. On 6 January 2013, Ronaldo captained his club for the first time. In May 2013, he scored his 200th goal for Real Madrid in a 6-2 win against Málaga. He finished as the Champions League top scorer that season. Ronaldo also won his second Ballon d'Or in 2013.\",\n", + " 'In the 2013-14 season, Ronaldo broke the record for most goals in one Champions League season by scoring his 17th goal with a penalty in extra time in the final against Atlético Madrid that Real Madrid won 4–1. The previous record was 14 goals, set by Messi in the 2011-12 season.',\n", + " \"In the 2014-15 season, Ronaldo set a new personal record by scoring 61 goals in all competitions. This achievement helped him win his second Ballon d'Or. He scored five goals in one match for the first time in his career in a 9-1 win vs. Granada on 5 April 2015. He became Real Madrid's all-time top scorer when he scored 5 goals against RCD Espanyol in a 6-0 away win on 12 September 2015. This brought his total goal tally to 230 goals in 203 games. The previous record holder was Raul.\",\n", + " \"On 18 April 2017, he became the first player to reach 100 goals in the UEFA Champions League, after he scored a hat-trick in a 4-2 extra-time win against Bayern Munich. On 18 March 2018, Ronaldo reached his 50th career hat-trick in a 6-3 win against Girona. Ronaldo scored an amazing bicycle-kick in a UEFA Champions League match against Juventus on 3 April 2018. He got a standing ovation, or round of applause, from the Juventus fans after scoring that goal. Real Madrid went on to play the final against Liverpool F.C.. Real Madrid became champions, so that was Ronaldo's 5th Champions League.\",\n", + " 'On 10 July 2018, He joined Juventus of Italy and signed a 4 year contract worth 112 Million Euros. The transfer was the highest paid for a player over 30 years old. People called it \"the deal of the century\".',\n", + " 'He scored his first goals for the club on 16 September against US Sassuolo. Juventus won 2-1 at home. Three days later, on 19 September, he was controversially sent off against Valencia C.F. for \"violent behavior\". He was crying as he received the red card and said he \"did nothing\". Ronaldo won his first trophy with the club, the 2018 Supercoppa Italiana, in January 2019. In the match, he scored the only goal from a header against AC Milan.',\n", + " \"Ronaldo began his international career with the Portugal under-15's in 2001. In 2002, he played the U-17 Euro with Portugal's U-17. In June 2003, Ronaldo won the Toulon Tournament with the Portugal U-20. He played with the U21's in 2003 as well during the 2004 U-21 Euro qualification stages. He also played with the Portugal under-23 team at the 2004 Summer Olympics, where he scored one goal against Morocco.\",\n", + " 'He kept progressing through the youth national teams until he played his first senior game for Portugal when he was 18 on 20 August 2003 against Kazakhstan. He scored his first goal for Portugal in a game against Greece at the UEFA Euro 2004.',\n", + " \"Ronaldo was selected to play at the 2006 FIFA World Cup, which was also his first World Cup. During the tournament, he scored a goal against Iran. That goal was also his first World Cup goal. In the match against England, Ronaldo's teammate at Manchester United, Wayne Rooney, was sent off. Ronaldo went up to the referee and appeared to ask the ref to give a red card to Rooney. Rooney pushed Ronaldo as he was talking to the ref. After Rooney left the pitch, Ronaldo was caught on camera winking at the Portugal bench. Even though the referee said Rooney would've gotten the red card if Ronaldo complained or not, these actions caused a lot of fans to hate Ronaldo because they believed he influenced the referee's decision.\",\n", + " \"Ronaldo became captain of Portugal for the first time in a friendly game against Brazil on 6 February 2007. At Euro 2008, he scored one goal against Czech Republic. Portugal were eliminated in the quarter-finals by barely losing to Germany 3-2. In the 2010 World Cup, Ronaldo only scored one goal in a 7-0 win against North Korea, but was the man of the match in all of Portugal's 3 group stage games. Portugal were eliminated in the round of 16 by Spain 1-0, and Spain went on to win the World Cup.\",\n", + " \"His first international hat-trick came in a 4-2 win against Northern Ireland on 6 September 2013. In that match, he scored 3 goals in 15 minutes. He became Portugal's all time top scorer when he scored twice against Cameroon in March 2014, with Portugal winning 5-1. At the 2014 World Cup, Ronaldo scored a goal against Ghana, and assisted a last minute equalizer in a 2-2 draw against the United States. In a 3-3 draw against Hungary in Euro 2016, Ronaldo scored a back heel goal. This goal made him the first player to score in four Euros. Although he had to leave the game early because he got injured in the UEFA Euro 2016 Final, Portugal still won 1-0 in extra time because of a goal from Eder.\",\n", + " \"In the 2018 World Cup held in Russia, Ronaldo scored 4 goals. In Portugal's first group game, he scored a hat trick against Spain in a 3-3 draw. After that he scored the winning goal against Morocco. In the last group game, he missed a penalty in a 1-1 draw against Iran. Portugal qualified to the knockout stages by finishing second in the group. They were eliminated by Uruguay 2-1. Ronaldo became one of only four players to score at 4 World Cups. He has played at 4 of them: 2006, 2010, 2014, and 2018.\",\n", + " 'In June 2019, Ronaldo won the UEFA Nations League with Portugal to give him his second international title. In the final, Portugal beat The Netherlands 1-0.',\n", + " 'Ronaldo is able to play on both wings and also as a striker since he is very strong with both feet, even though he is naturally right footed. He is also one of the world\\'s fastest players. He has good heading ability because he is over 6 feet tall and jumps high. He is also known for his powerful \"knuckleball\" free kicks. The \"knuckleball\" technique is when the ball spins very little and creates an unpredictable motion. He combines this with his powerful shot, making it hard for goalkeepers to stop his shots. Ronaldo has also been known for his dribbling, as he likes to do many tricks and with the ball to pass defenders, such as the . When he was at Manchester United, he would play as a winger and try to send into the middle. At Real Madrid, he changed his playing style by moving more towards the middle and becoming more of a striker. He also focused more on scoring goals. When he arrived to Juventus, he stayed with this playing style of being a goal scorer and target man, although he dribbled with the ball more because he sometimes liked to play his traditional winger position and go one-on-one with defenders. He also sent crosses more frequently than he did in his last few seasons at Real Madrid.',\n", + " 'Ronaldo has been criticized for \"diving\" by many people, including his Manchester United manager Alex Ferguson. He has also been criticized for being arrogant, such as complaining for not receiving set-pieces (free-kicks and penalties) when he gets fouled, having too much self-confidence, not celebrating with teammates after scoring goals, and getting excessively angry with others after losing. Examples of this are when he threw a reporter\\'s microphone into a lake before a UEFA Euro 2016 match, and negative comments made at the Iceland national team after playing against them.',\n", + " \"Ronaldo's father, José Aveiro, died of liver disease at age 52 in September 2005. Ronaldo was 20 years old at the time. Ronaldo said that he does not like to drink alcohol, mostly because of his dad's death, but has on some very few occasions.\",\n", + " 'In 2006, Ronaldo opened his first fashion boutique under the name \"CR7\" (his initials and shirt number) on the\\xa0island he was born in, Madeira. He opened a second boutique in Lisbon in 2008, and a third in 2009, located in Madrid. In December 2013, Ronaldo opened his own museum called \"Museu CR7\", which has all of his trophies and awards from his career.',\n", + " \"Ronaldo became a father on 17 June 2010. He had a son, named Cristiano Jr.\\xa0He was born in the United States through an American surrogate he met in a restaurant, and Ronaldo announced that he had full custody.\\xa0Ronaldo has never publicly revealed information about his son's mother, but he says he will reveal it to Cristiano Jr. when he gets older. On 8 June 2017, Ronaldo confirmed on social media that he had become the father to twins, Mateo and Eva. They were born in the United States to a mother. In November 2017, his girlfriend Georgina Rodriguez gave birth to their first daughter, Alana.\",\n", + " 'Ronaldo has had many relationships. He was in a relationship with Russian model Irina Shayk from 2010–2015. In 2016, he began to date Spanish model Georgina Rodriguez. They were publicly seen for the first time at Disneyland Paris in November 2016.',\n", + " 'Ronaldo is a\\xa0Roman Catholic. He does not have\\xa0tattoos\\xa0because it would prevent him from donating blood. On 9 November 2015, a movie about his lifestyle and his career was released. The title of the movie is \"Ronaldo\".',\n", + " 'On 29 March 2017, Madeira Airport was renamed to Cristiano Ronaldo International Airport. A bust of Ronaldo was also revealed as part of the official renaming ceremony.',\n", + " 'He is currently the most followed Instagram user, with over 200 million followers as of February 2020. He passed Selena Gomez as the most followed person in October 2018, with 144 million followers.',\n", + " 'In June 2018, Ronaldo was given a suspended jail sentence of 2 years and a fine of €18.8 million for tax evasion.',\n", + " 'In 2017, a woman claimed she was raped by Ronaldo at a hotel in Las Vegas in June 2009. Many articles state that Ronaldo and the woman both met each other at the nightclub of the hotel. Ronaldo later invited her to his suite, and that was where the rape occurred. Ronaldo paid that woman $375,000 to stay quiet, and an agreement was made between Ronaldo\\'s lawyers and the woman\\'s lawyers that if she publicly shared information of what happened, she had to pay those $375,000 back. Ronaldo himself denies raping the woman and calls it \"fake news\". In July 2019, prosecutors said they would not charge Ronaldo because there was not enough evidence.'],\n", + " 'documentid': 'id:wikipedia:wiki::65655',\n", + " 'title': 'Cristiano Ronaldo',\n", + " 'url': 'https://simple.wikipedia.org/wiki?curid=65655'}}]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "result = app.query(body={\n", " 'yql': 'select * from wiki where userQuery()',\n", " 'query': 24,\n", - " 'ranking.profile': 'bm25'\n", + " 'ranking.profile': 'bm25',\n", + " 'hits': 2\n", "})\n", "result.hits" ] @@ -253,15 +410,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "3bccdb4c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'id:wikipedia:wiki::9985',\n", + " 'relevance': 0.8807156260391702,\n", + " 'source': 'wiki_content',\n", + " 'fields': {'matchfeatures': {'closest(paragraph_embeddings)': {'type': 'tensor(p{})',\n", + " 'cells': {'4': 1.0}}},\n", + " 'sddocname': 'wiki',\n", + " 'paragraphs': ['The 24-hour clock is a way of telling the time in which the day runs from midnight to midnight and is divided into 24 hours, numbered from 0 to 23. It does not use a.m. or p.m. This system is also referred to (only in the US and the English speaking parts of Canada) as military time or (only in the United Kingdom and now very rarely) as continental time. In some parts of the world, it is called railway time. Also, the international standard notation of time (ISO 8601) is based on this format.',\n", + " 'A time in the 24-hour clock is written in the form hours:minutes (for example, 01:23), or hours:minutes:seconds (01:23:45). Numbers under 10 have a zero in front (called a leading zero); e.g. 09:07. Under the 24-hour clock system, the day begins at midnight, 00:00, and the last minute of the day begins at 23:59 and ends at 24:00, which is identical to 00:00 of the following day. 12:00 can only be mid-day. Midnight is called 24:00 and is used to mean the end of the day and 00:00 is used to mean the beginning of the day. For example, you would say \"Tuesday at 24:00\" and \"Wednesday at 00:00\" to mean exactly the same time.',\n", + " 'However, the US military prefers not to say 24:00 - they do not like to have two names for the same thing, so they always say \"23:59\", which is one minute before midnight.',\n", + " '24-hour clock time is used in computers, military, public safety, and transport. In many Asian, European and Latin American countries people use it to write the time. Many European people use it in speaking.',\n", + " 'In railway timetables 24:00 means the \"end\" of the day. For example, a train due to arrive at a station during the last minute of a day arrives at 24:00; but trains which depart during the first minute of the day go at 00:00.'],\n", + " 'documentid': 'id:wikipedia:wiki::9985',\n", + " 'title': '24-hour clock',\n", + " 'url': 'https://simple.wikipedia.org/wiki?curid=9985'}}]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "result = app.query(body={\n", " 'yql': 'select * from wiki where {targetHits:1}nearestNeighbor(paragraph_embeddings,q)',\n", " 'input.query(q)': 'embed(what does 24 mean in the context of railways)',\n", - " 'ranking.profile': 'semantic'\n", + " 'ranking.profile': 'semantic',\n", + " 'hits': 2\n", "})\n", "result.hits" ] @@ -294,10 +476,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "4ad0d11f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'id:wikipedia:wiki::9985',\n", + " 'relevance': 4.163377257526372,\n", + " 'source': 'wiki_content',\n", + " 'fields': {'matchfeatures': {'bm25(paragraphs)': 10.468827250036052,\n", + " 'bm25(title)': 1.1272217840066165,\n", + " 'closest(paragraph_embeddings)': {'type': 'tensor(p{})',\n", + " 'cells': {'4': 1.0}},\n", + " 'firstPhase': 0.8807156260391702,\n", + " 'all_paragraph_similarities': {'type': 'tensor(p{})',\n", + " 'cells': {'1': 0.8061168789863586,\n", + " '2': 0.7993348240852356,\n", + " '3': 0.8240271806716919,\n", + " '4': 0.880715548992157,\n", + " '0': 0.8497915267944336}},\n", + " 'avg_paragraph_similarity': 0.8319971919059753,\n", + " 'max_paragraph_similarity': 0.880715548992157},\n", + " 'sddocname': 'wiki',\n", + " 'paragraphs': ['The 24-hour clock is a way of telling the time in which the day runs from midnight to midnight and is divided into 24 hours, numbered from 0 to 23. It does not use a.m. or p.m. This system is also referred to (only in the US and the English speaking parts of Canada) as military time or (only in the United Kingdom and now very rarely) as continental time. In some parts of the world, it is called railway time. Also, the international standard notation of time (ISO 8601) is based on this format.',\n", + " 'A time in the 24-hour clock is written in the form hours:minutes (for example, 01:23), or hours:minutes:seconds (01:23:45). Numbers under 10 have a zero in front (called a leading zero); e.g. 09:07. Under the 24-hour clock system, the day begins at midnight, 00:00, and the last minute of the day begins at 23:59 and ends at 24:00, which is identical to 00:00 of the following day. 12:00 can only be mid-day. Midnight is called 24:00 and is used to mean the end of the day and 00:00 is used to mean the beginning of the day. For example, you would say \"Tuesday at 24:00\" and \"Wednesday at 00:00\" to mean exactly the same time.',\n", + " 'However, the US military prefers not to say 24:00 - they do not like to have two names for the same thing, so they always say \"23:59\", which is one minute before midnight.',\n", + " '24-hour clock time is used in computers, military, public safety, and transport. In many Asian, European and Latin American countries people use it to write the time. Many European people use it in speaking.',\n", + " 'In railway timetables 24:00 means the \"end\" of the day. For example, a train due to arrive at a station during the last minute of a day arrives at 24:00; but trains which depart during the first minute of the day go at 00:00.'],\n", + " 'documentid': 'id:wikipedia:wiki::9985',\n", + " 'title': '24-hour clock',\n", + " 'url': 'https://simple.wikipedia.org/wiki?curid=9985'}}]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "result = app.query(body={\n", " 'yql': 'select * from wiki where userQuery() or ({targetHits:1}nearestNeighbor(paragraph_embeddings,q))',\n", @@ -340,10 +557,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "250f2c1b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'id:wikipedia:wiki::9985',\n", + " 'relevance': 4.307057297582033,\n", + " 'source': 'wiki_content',\n", + " 'fields': {'matchfeatures': {'bm25(paragraphs)': 10.468827250036052,\n", + " 'bm25(title)': 1.1272217840066165,\n", + " 'closest(paragraph_embeddings)': {'type': 'tensor(p{})',\n", + " 'cells': {'4': 1.0}},\n", + " 'firstPhase': 0.8807156260391702,\n", + " 'all_paragraph_similarities': {'type': 'tensor(p{})',\n", + " 'cells': {'1': 0.8061168789863586,\n", + " '2': 0.7993348240852356,\n", + " '3': 0.8240271806716919,\n", + " '4': 0.880715548992157,\n", + " '0': 0.8497915267944336}},\n", + " 'avg_paragraph_similarity': 0.8319971919059753,\n", + " 'max_paragraph_similarity': 0.880715548992157},\n", + " 'sddocname': 'wiki',\n", + " 'paragraphs': ['The 24-hour clock is a way of telling the time in which the day runs from midnight to midnight and is divided into 24 hours, numbered from 0 to 23. It does not use a.m. or p.m. This system is also referred to (only in the US and the English speaking parts of Canada) as military time or (only in the United Kingdom and now very rarely) as continental time. In some parts of the world, it is called railway time. Also, the international standard notation of time (ISO 8601) is based on this format.',\n", + " 'A time in the 24-hour clock is written in the form hours:minutes (for example, 01:23), or hours:minutes:seconds (01:23:45). Numbers under 10 have a zero in front (called a leading zero); e.g. 09:07. Under the 24-hour clock system, the day begins at midnight, 00:00, and the last minute of the day begins at 23:59 and ends at 24:00, which is identical to 00:00 of the following day. 12:00 can only be mid-day. Midnight is called 24:00 and is used to mean the end of the day and 00:00 is used to mean the beginning of the day. For example, you would say \"Tuesday at 24:00\" and \"Wednesday at 00:00\" to mean exactly the same time.',\n", + " 'However, the US military prefers not to say 24:00 - they do not like to have two names for the same thing, so they always say \"23:59\", which is one minute before midnight.',\n", + " '24-hour clock time is used in computers, military, public safety, and transport. In many Asian, European and Latin American countries people use it to write the time. Many European people use it in speaking.',\n", + " 'In railway timetables 24:00 means the \"end\" of the day. For example, a train due to arrive at a station during the last minute of a day arrives at 24:00; but trains which depart during the first minute of the day go at 00:00.'],\n", + " 'documentid': 'id:wikipedia:wiki::9985',\n", + " 'title': '24-hour clock',\n", + " 'url': 'https://simple.wikipedia.org/wiki?curid=9985'}}]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "result = app.query(body={\n", " 'yql': 'select * from wiki where url contains \"9985\" and userQuery() or ({targetHits:1}nearestNeighbor(paragraph_embeddings,q))',\n", @@ -365,7 +617,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "8ca46387", "metadata": {}, "outputs": [],