offline-twitter/scraper/tweet_test.go

package scraper_test

import (
	"encoding/json"
	"io/ioutil"
	"testing"

    "github.com/stretchr/testify/assert"
    "github.com/stretchr/testify/require"

	. "offline_twitter/scraper"
)

func load_tweet_from_file(filename string) Tweet{
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	var apitweet APITweet
	err = json.Unmarshal(data, &apitweet)
	if err != nil {
		panic(err)
	}
	tweet, err := ParseSingleTweet(apitweet)
	if err != nil {
		panic(err)
	}
	return tweet
}


func TestParseSingleTweet(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_unicode_chars.json")

	assert.Equal("The fact that @michaelmalice new book ‘The Anarchist Handbook’ is just absolutely destroying on the charts is the largest white pill I’ve swallowed in years.", tweet.Text)
	assert.Len(tweet.Mentions, 1)
	assert.Contains(tweet.Mentions, UserHandle("michaelmalice"))
	assert.Empty(tweet.Urls)
	assert.Equal(int64(1621639105), tweet.PostedAt.Unix())
	assert.Zero(tweet.QuotedTweetID)
	assert.Empty(tweet.Polls)
}

func TestParseTweetWithImage(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_image.json")

	assert.Equal("this saddens me every time", tweet.Text)
	assert.Len(tweet.Images, 1)
}

func TestParseTweetWithQuotedTweetAsLink(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_quoted_tweet_as_link2.json")

	assert.Equal("sometimes they're too dimwitted to even get the wrong title right", tweet.Text)
	assert.Equal(TweetID(1395882872729477131), tweet.InReplyToID)
	assert.Equal(TweetID(1396194494710788100), tweet.QuotedTweetID)
	assert.Empty(tweet.ReplyMentions)
	assert.Empty(tweet.Polls)
}

func TestParseTweetWithVideo(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_video.json")
	assert.Empty(tweet.Images)
	assert.Len(tweet.Videos, 1)

	v := tweet.Videos[0]
	assert.Equal("https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/720x1280/sm4iL9_f8Lclh0aa.mp4?tag=12", v.RemoteURL)
	assert.False(v.IsGif)
}

func TestParseTweetWithGif(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json")
	assert.Len(tweet.Videos, 1)

	v := tweet.Videos[0]
	assert.Equal("https://video.twimg.com/tweet_video/E189-VhVoAYcrDv.mp4", v.RemoteURL)
	assert.True(v.IsGif)
}

func TestParseTweetWithUrl(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_url_card.json")
	assert.Empty(tweet.Polls)
	assert.Len(tweet.Urls, 1)

	u := tweet.Urls[0]
	assert.Equal("https://reason.com/2021/08/30/la-teachers-union-cecily-myart-cruz-learning-loss/", u.Text)
	assert.True(u.HasCard)
	assert.Equal("reason.com", u.Domain)
}

func TestParseTweetWithUrlButNoCard(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_url_but_no_card.json")
	assert.Len(tweet.Urls, 1)

	u := tweet.Urls[0]
	assert.Equal("https://www.politico.com/newsletters/west-wing-playbook/2021/09/16/the-jennifer-rubin-wh-symbiosis-494364", u.Text)
	assert.False(u.HasCard)
}

func TestParseTweetWithMultipleUrls(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_multiple_urls.json")
	assert.Empty(tweet.Polls)
	assert.Len(tweet.Urls, 3)

	assert.False(tweet.Urls[0].HasCard)
	assert.False(tweet.Urls[1].HasCard)
	assert.True (tweet.Urls[2].HasCard)

	assert.Equal("Biden’s victory came from the suburbs", tweet.Urls[2].Title)
}

func TestTweetWithLotsOfReplyMentions(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_at_mentions_in_front.json")
	assert.Len(tweet.ReplyMentions, 4)

	for i, v := range []UserHandle{"rob_mose", "primalpoly", "jmasseypoet", "SpaceX"} {
		assert.Equal(v, tweet.ReplyMentions[i])
	}
}

func TestTweetWithPoll(t *testing.T) {
	assert := assert.New(t)
	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_poll_4_choices.json")
	assert.Len(tweet.Polls, 1)

	p := tweet.Polls[0]
	assert.Equal(tweet.ID, p.TweetID)
	assert.Equal(4, p.NumChoices)
	assert.Equal("Tribal armband", p.Choice1)
	assert.Equal("Marijuana leaf", p.Choice2)
	assert.Equal("Butterfly", p.Choice3)
	assert.Equal("Maple leaf", p.Choice4)
	assert.Equal(1593, p.Choice1_Votes)
	assert.Equal(624, p.Choice2_Votes)
	assert.Equal(778, p.Choice3_Votes)
	assert.Equal(1138, p.Choice4_Votes)
	assert.Equal(1440 * 60, p.VotingDuration)
	assert.Equal(int64(1638331934), p.VotingEndsAt.Unix())
	assert.Equal(int64(1638331935), p.LastUpdatedAt.Unix())
}


func TestParseTweetResponse(t *testing.T) {
	assert := assert.New(t)
	data, err := ioutil.ReadFile("test_responses/michael_malice_feed.json")
	if err != nil {
		panic(err)
	}
	var tweet_resp TweetResponse
	err = json.Unmarshal(data, &tweet_resp)
	require.NoError(t, err)

	tweets, retweets, users, err := ParseTweetResponse(tweet_resp)
	require.NoError(t, err)

	assert.Len(tweets, 29 - 3)
	assert.Len(retweets, 3)
	assert.Len(users, 9)
}

func TestParseTweetResponseWithTombstones(t *testing.T) {
	assert := assert.New(t)
	data, err := ioutil.ReadFile("test_responses/tombstones/tombstone_deleted.json")
	if err != nil {
		panic(err)
	}
	var tweet_resp TweetResponse
	err = json.Unmarshal(data, &tweet_resp)
	require.NoError(t, err)

	extra_users := tweet_resp.HandleTombstones()
	assert.Len(extra_users, 1)

	tweets, retweets, users, err := ParseTweetResponse(tweet_resp)
	require.NoError(t, err)

	assert.Len(tweets, 2)
	assert.Len(retweets, 0)
	assert.Len(users, 1)
}